diff --git a/.gitea/workflows/build-test-deploy.yml b/.gitea/workflows/build-test-deploy.yml index f75af564..947f7ec7 100644 --- a/.gitea/workflows/build-test-deploy.yml +++ b/.gitea/workflows/build-test-deploy.yml @@ -64,27 +64,54 @@ jobs: with: fetch-depth: 0 + - name: Validate NuGet restore source ordering + run: python3 ops/devops/validate_restore_sources.py + - name: Setup .NET ${{ env.DOTNET_VERSION }} uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true - - name: Restore dependencies - run: dotnet restore src/StellaOps.Feedser.sln + - name: Restore Concelier solution + run: dotnet restore src/StellaOps.Concelier.sln - - name: Build solution (warnings as errors) - run: dotnet build src/StellaOps.Feedser.sln --configuration $BUILD_CONFIGURATION --no-restore -warnaserror + - name: Build Concelier solution (warnings as errors) + run: dotnet build src/StellaOps.Concelier.sln --configuration $BUILD_CONFIGURATION --no-restore -warnaserror - - name: Run unit and integration tests + - name: Run Concelier unit and integration tests run: | mkdir -p "$TEST_RESULTS_DIR" - dotnet test src/StellaOps.Feedser.sln \ + dotnet test src/StellaOps.Concelier.sln \ --configuration $BUILD_CONFIGURATION \ --no-build \ - --logger "trx;LogFileName=stellaops-feedser-tests.trx" \ + --logger "trx;LogFileName=stellaops-concelier-tests.trx" \ --results-directory "$TEST_RESULTS_DIR" + - name: Lint policy DSL samples + run: dotnet run --project tools/PolicyDslValidator/PolicyDslValidator.csproj -- --strict docs/examples/policies/*.yaml + + - name: Run policy simulation smoke tests (first pass) + run: dotnet run --project tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj -- --scenario-root samples/policy/simulations --output artifacts/policy-simulations/run1 + + - name: Verify policy simulation determinism + run: | + dotnet run --project tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj -- --scenario-root samples/policy/simulations --output artifacts/policy-simulations/run2 + diff -u \ + artifacts/policy-simulations/run1/policy-simulation-summary.json \ + artifacts/policy-simulations/run2/policy-simulation-summary.json + + - name: Upload policy simulation artifacts + uses: actions/upload-artifact@v4 + with: + name: policy-simulation-diffs + path: artifacts/policy-simulations + if-no-files-found: error + retention-days: 14 + + - name: Run release tooling tests + run: python ops/devops/release/test_verify_release.py + - name: Build scanner language analyzer projects run: | dotnet restore src/StellaOps.sln diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 068728c7..63f091f6 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -44,6 +44,9 @@ jobs: with: fetch-depth: 0 + - name: Validate NuGet restore source ordering + run: python3 ops/devops/validate_restore_sources.py + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -58,6 +61,75 @@ jobs: dotnet-version: ${{ env.DOTNET_VERSION }} include-prerelease: true + - name: Install cross-arch objcopy tooling + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends binutils-aarch64-linux-gnu + + - name: Publish Python analyzer plug-in + run: | + set -euo pipefail + dotnet publish src/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj \ + --configuration Release \ + --output out/analyzers/python \ + --no-self-contained + mkdir -p plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python + cp out/analyzers/python/StellaOps.Scanner.Analyzers.Lang.Python.dll plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/ + if [ -f out/analyzers/python/StellaOps.Scanner.Analyzers.Lang.Python.pdb ]; then + cp out/analyzers/python/StellaOps.Scanner.Analyzers.Lang.Python.pdb plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/ + fi + + - name: Run Python analyzer smoke checks + run: | + dotnet run \ + --project tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj \ + --configuration Release \ + -- \ + --repo-root . + + # Note: this step enforces DEVOPS-REL-14-004 by signing the restart-only Python plug-in. + # Ensure COSIGN_KEY_REF or COSIGN_IDENTITY_TOKEN is configured, otherwise the job will fail. + - name: Sign Python analyzer artefacts + env: + COSIGN_KEY_REF: ${{ secrets.COSIGN_KEY_REF }} + COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} + COSIGN_IDENTITY_TOKEN: ${{ secrets.COSIGN_IDENTITY_TOKEN }} + run: | + set -euo pipefail + if [[ -z "${COSIGN_KEY_REF:-}" && -z "${COSIGN_IDENTITY_TOKEN:-}" ]]; then + echo "::error::COSIGN_KEY_REF or COSIGN_IDENTITY_TOKEN must be provided to sign analyzer artefacts." >&2 + exit 1 + fi + + export COSIGN_PASSWORD="${COSIGN_PASSWORD:-}" + export COSIGN_EXPERIMENTAL=1 + + PLUGIN_DIR="plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python" + ARTIFACTS=( + "StellaOps.Scanner.Analyzers.Lang.Python.dll" + "manifest.json" + ) + + for artifact in "${ARTIFACTS[@]}"; do + FILE="${PLUGIN_DIR}/${artifact}" + if [[ ! -f "${FILE}" ]]; then + echo "::error::Missing analyzer artefact ${FILE}" >&2 + exit 1 + fi + + sha256sum "${FILE}" | awk '{print $1}' > "${FILE}.sha256" + + SIGN_ARGS=(--yes "${FILE}") + if [[ -n "${COSIGN_KEY_REF:-}" ]]; then + SIGN_ARGS=(--key "${COSIGN_KEY_REF}" "${SIGN_ARGS[@]}") + fi + if [[ -n "${COSIGN_IDENTITY_TOKEN:-}" ]]; then + SIGN_ARGS=(--identity-token "${COSIGN_IDENTITY_TOKEN}" "${SIGN_ARGS[@]}") + fi + + cosign sign-blob "${SIGN_ARGS[@]}" > "${FILE}.sig" + done + - name: Install Helm 3.16.0 run: | curl -fsSL https://get.helm.sh/helm-v3.16.0-linux-amd64.tar.gz -o /tmp/helm.tgz @@ -124,6 +196,7 @@ jobs: mkdir -p out/release - name: Build release bundle + # NOTE (DEVOPS-REL-17-004): build_release.py now fails if out/release/debug is missing env: COSIGN_KEY_REF: ${{ secrets.COSIGN_KEY_REF }} COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} @@ -141,6 +214,10 @@ jobs: --git-sha "${{ github.sha }}" \ "${EXTRA_ARGS[@]}" + - name: Verify release artefacts + run: | + python ops/devops/release/verify_release.py --release-dir out/release + - name: Upload release artefacts uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index d44a4a1d..31af354d 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,8 @@ seed-data/cert-bund/**/*.sha256 out/offline-kit/web/**/* src/StellaOps.Web/node_modules/**/* -src/StellaOps.Web/.angular/**/* \ No newline at end of file +src/StellaOps.Web/.angular/**/* +**/node_modules/**/* +node_modules +node_modules +node_modules diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 00000000..c2475e91 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,12 @@ + + + $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)')) + $([System.IO.Path]::GetFullPath('$(StellaOpsRepoRoot)local-nuget/')) + https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json + https://api.nuget.org/v3/index.json + <_StellaOpsDefaultRestoreSources>$(StellaOpsLocalNuGetSource);$(StellaOpsDotNetPublicSource);$(StellaOpsNuGetOrgSource) + <_StellaOpsOriginalRestoreSources Condition="'$(_StellaOpsOriginalRestoreSources)' == ''">$(RestoreSources) + $(_StellaOpsDefaultRestoreSources) + $(_StellaOpsDefaultRestoreSources);$(_StellaOpsOriginalRestoreSources) + + diff --git a/EPIC_3.md b/EPIC_3.md deleted file mode 100644 index 5d7d499b..00000000 --- a/EPIC_3.md +++ /dev/null @@ -1,531 +0,0 @@ -Here’s Epic 3 in the same “paste‑into‑repo” format: exhaustive, implementation‑ready, and aligned with the AOC model plus the Policy Engine from the previous epics. - ---- - -# Epic 3: StellaOps Console (Web UI over WebServices) - -> Short name: **StellaOps Console** -> Services touched: **Web API Gateway, Authority (authN/Z), Policy Engine, SBOM Service, Conseiller (Feedser), Excitator (Vexer), Scheduler/Workers, Telemetry** -> Data stores used via APIs: **MongoDB (reads only from UI), object storage for traces**, optional **Redis/NATS** for live updates -> Deliverable: **TypeScript/React web application** with a component library and feature modules, packaged as container images and static builds - ---- - -## 1) What it is - -**StellaOps Console** is the first‑party Web UI for all Stella WebServices. It provides a cohesive, role‑aware surface for: - -* Viewing raw AOC facts (advisories, VEX, SBOMs) without mutation. -* Applying and simulating policies (VEX application rules, advisory normalization) then exploring **effective findings**. -* Navigating SBOMs as graphs, zooming into components, and seeing linked advisories/VEX with clear precedence. -* Running and monitoring evaluations, auditing why decisions were made, and exporting evidence. -* Administering tenants, users and roles, API tokens, and integrations. -* Publishing a self‑hostable Console image and a **Download & Install** page covering all product containers. - -The Console is a read‑write client for allowable operations (policy authoring, run orchestration, approvals), and strictly read‑only for **raw facts** per the AOC enforcement. It is **not** a new API; it is a UI over the existing ones with strong guardrails and deterministic behavior. - ---- - -## 2) Why - -* Teams need a single, consistent interface to explore SBOMs, advisories, VEX, and policy outcomes. -* Audits require visible provenance, replayable evidence, and explanation chains. -* Policy creation and simulation are safer when you can see deltas and traces. -* Many workflows benefit from visual tools: graph explorers, diff views, and step‑wise wizards. -* Not everyone wants to live in the CLI all day. Parity and choice matter. - ---- - -## 3) How it should work (maximum detail) - -### 3.1 Information architecture - -Top‑level navigation with a tenant context picker: - -1. **Dashboard**: high‑level posture and recent changes. -2. **SBOMs**: catalog, search, and **Graph Explorer**. -3. **Advisories & VEX**: raw fact browsers with aggregation‑not‑merge semantics. -4. **Findings**: policy‑materialized findings with filters and explanations. -5. **Policies**: editor, simulation, versioning, approvals. -6. **Runs**: orchestration, live progress, history, diffs. -7. **Reports & Export**: evidence packages, CSV/JSON exports. -8. **Admin**: users/roles, tokens, SSO, tenants, registries, settings. -9. **Downloads**: product containers and installation instructions. - -Global elements: - -* **Global Filters**: policy version, environment profile, severity band, time window. -* **Search Bar**: PURL, CVE/GHSA IDs, SBOM IDs. -* **Live Status**: background jobs, queue lag, last sync cursors. -* **Help & Docs**: contextual deep links into `/docs/*`. - -### 3.2 Navigation & routes - -``` -/dashboard -/sboms -/sboms/:sbomId -/sboms/:sbomId/graph -/advisories -/advisories/:advisoryId (shows all linked sources; aggregation only) -/vex -/vex/:vexId -/findings?policy=:pid&sbom=:sid&status=:st&severity=:sev -/findings/:findingId/explain -/policies -/policies/:policyId/versions/:v -/policies/:policyId/simulate -/runs -/runs/:runId -/reports -/admin/users -/admin/roles -/admin/tenants -/admin/integrations -/admin/tokens -/downloads -``` - -### 3.3 Core feature modules - -#### 3.3.1 Dashboard - -* Cards: “Findings by severity,” “VEX overrides in last 24h,” “New advisories linked,” “Run health,” “Policy changes.” -* Click‑through to filtered views. -* Data sources: aggregated endpoints exposed by Web API (no client‑side aggregation over large sets). - -#### 3.3.2 SBOM Explorer (catalog + graph) - -* **Catalog**: table with SBOM ID, artifact name/version, source, ingest time, component count, last evaluation per policy. -* **Detail**: components tabular view with paging; filters by package type, license, scope. -* **Graph Explorer**: - - * Interactive canvas with pan/zoom, focus on component, dependency paths, reachability placeholders. - * Overlay toggles: highlight components with affected findings; show VEX “not_affected” zones; show licenses risk overlay. - * **Policy overlays**: toggle between policy versions to see in‑place severity/status changes. -* **Actions**: export component list, copy PURL, open related findings. - -**AOC alignment**: SBOM content is immutable; any edits are proposed as new SBOM versions upstream. UI displays raw SBOM JSON in a read‑only side panel. - -#### 3.3.3 Advisories & VEX browsers - -* **Advisories list**: - - * Left panel: filters by source (OSV, GHSA, CSAF vendors, NVD), published/modified time, affected ecosystem. - * Middle panel: **aggregation group** keyed by linkset identity (same vulnerability across sources). No merging; show a roll‑up with per‑source chips. - * Right panel: selected advisory source view with raw JSON, references, CVSS vectors, and “linked SBOM components” sample. - * Severity shown three ways: vendor‑reported, normalized (per mapping), and **effective** under the currently selected policy. -* **VEX list**: - - * Filters by vendor, product, status, justification, scope. - * Detail panel: all statements applying to the same `(component, advisory)` tuple, with precedence logic visualization and the statement that won under the current policy. - * Raw JSON viewer for each document. - -**Strict rule**: Conseiller and Excitator are visualized as **aggregators only**. No UI affordance suggests server‑side merging. All links route to raw documents with reference IDs. - -#### 3.3.4 Findings - -* Virtualized table supporting millions of rows via server‑side pagination and cursoring. -* Columns: policy, SBOM, component PURL, advisory IDs (chips for each source), status, severity, last updated, rationale count. -* Row click → **Explain** view with rule hits in order, references to advisories/VEX used, and links to raw docs and trace blobs. -* Bulk export with query replay (the export API re‑runs the same filters on the server and streams CSV/JSON). - -#### 3.3.5 Policies - -* Embedded **Policy Editor** (from Epic 2) with Monaco features, simulation panel, diffs, and approval workflow. -* Pre‑commit lint and compile; cannot submit with syntax errors. -* Simulation results show increase/decrease unchanged counts, top rules impacting results, and sample affected components. - -#### 3.3.6 Runs - -* Queue view: queued/running/succeeded/failed with timestamps and SLA hints. -* Live progress with **SSE/WebSocket** updates: tuples processed, rules fired, findings materialized. -* Diff view between runs for the same policy and SBOM set. -* Retry and cancel actions as allowed by RBAC. - -#### 3.3.7 Reports & Export - -* Evidence bundle creation: include policy version, run ID, sample traces, and result slices. -* Export templates (CSV for management, JSON/NDJSON for SIEM ingestion). -* Signed export manifests with checksums. - -#### 3.3.8 Admin - -* Users & Roles: invite, disable, role mapping. -* Tenants: create, switch, default policy bindings. -* Tokens: create scoped API tokens with expirations. -* Integrations: configure SSO (OIDC), registries, webhooks. -* Settings: environment defaults for policy evaluation (exposure, runtime hints). - -#### 3.3.9 Downloads - -* List of official Docker images: `stella-console`, `stella-api`, `conseiller`, `excitator`, `sbom-svc`, `policy-engine`, etc. -* Version matrix, pull commands, Helm chart snippet, offline tarballs, and system requirements. -* Link to `/docs/install/docker.md` and `/docs/deploy/console.md`. - -### 3.4 UX flows (key tasks) - -* **Triage a vulnerability**: search CVE → open roll‑up → view all sources → jump to affected findings → open Explain → see VEX precedence → decide if policy change is needed → simulate policy → if good, submit and request approval → run → verify new findings. -* **Investigate SBOM**: open SBOM → Graph Explorer → highlight affected nodes under policy P‑X vN → click a node → see linked advisories + VEX → open Explain for a specific finding. -* **Audit evidence**: open run → download evidence bundle with policy, run metadata, traces, and effective finding slice. -* **Onboard team**: invite users, set roles, define default tenant policies, give read‑only access to auditors. - -### 3.5 CLI vs UI parity - -Create `/docs/cli-vs-ui-parity.md` with a matrix. Principle: - -* All **read** capabilities must exist in both CLI and UI. -* All **policy lifecycle** actions exist in both. -* Long‑running operations can be initiated in UI and monitored in either surface. - -### 3.6 Security & auth - -* Auth: OIDC with PKCE; short‑lived ID tokens; silent refresh. -* RBAC enforced by the API; UI only gates affordances and never trusts itself. -* CSRF not applicable for token‑based APIs; still set robust **CSP**, **X‑Frame‑Options**, and **Referrer‑Policy**. -* Tenancy: every API call includes tenant header; UI shows explicit tenant badge. -* Sensitive pages require **fresh auth** (re‑prompt). - -> **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. - -### 3.7 Accessibility & i18n - -* WCAG 2.1 AA: keyboard nav, focus indicators, ARIA for tables and graphs, color‑contrast tests. -* i18n scaffolding via ICU messages; English shipped first; content keys stored in code, translations as JSON resources. - -> **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. - -### 3.8 Performance - -* Use server‑side pagination and cursoring everywhere; never fetch unbounded lists. -* Virtualized tables and lazy panels. -* Graph Explorer loads neighborhood windows, not whole graphs. -* Cache with TanStack Query; deduplicate requests; stale‑while‑revalidate. -* Performance budgets in CI (Lighthouse): TTI < 3.5s on reference hardware. - -### 3.9 Error handling & offline - -* Error boundaries per feature; retry buttons; copyable request IDs. -* Network loss → banner + read‑only cached views where safe. -* Clear messages for **AOC** constraints: raw facts cannot be edited. - -### 3.10 Telemetry & observability - -* UI event telemetry to internal sink (no third‑party beacons by default). -* Metrics: UI API latency percentiles, error rates, SSE subscription health. -* Feature flags to dark‑launch modules. - ---- - -## 4) APIs consumed (representative) - -* `GET /sboms`, `GET /sboms/{id}`, `GET /sboms/{id}/components?cursor=...` -* `GET /advisories?source=...`, `GET /advisories/{id}`, `GET /advisories/{id}/linked` -* `GET /vex?status=...`, `GET /vex/{id}` -* `GET /findings/{policyId}` and `GET /findings/{policyId}/{findingId}/explain` -* `POST /policies`, `POST /policies/{id}/compile`, `POST /policies/{id}/simulate`, `POST /policies/{id}/approve` -* `POST /policies/{id}/runs`, `GET /policies/{id}/runs/{runId}` with SSE for progress -* `POST /exports` for evidence bundles -* `GET /auth/user`, `GET /auth/tenants`, `POST /admin/users`, `POST /admin/tokens` - -All calls include tenant scope headers and bearer tokens from Authority. - ---- - -## 5) Implementation plan - -### 5.1 Frontend architecture - -* **Framework**: Next.js 14 (App Router) with TypeScript. -* **State/data**: TanStack Query for server state; Redux only if a global app state proves necessary. -* **UI toolkit**: Internal **Stella UI** component library (headless + primitives) with CSS variables and design tokens. -* **Visualization**: D3 for graph, Monaco for policy editing. -* **Testing**: Playwright (E2E), Vitest/Jest (unit), Storybook (components), Lighthouse (perf). -* **i18n**: `@formatjs/intl` + message catalogs. -* **Packaging**: static build served by Node adapter behind the API gateway; also a `stella-console` Docker image. - -**Repo layout** - -``` -/console - /apps/web - /packages/ui # design system & components - /packages/api # typed API clients (OpenAPI codegen) - /packages/features # feature modules (sboms, advisories, vex, findings, policies, runs, admin) - /packages/utils - /e2e - /storybook -``` - -### 5.2 Design System (packages/ui) - -* Foundation tokens: color, spacing, typography, elevation; dark/light modes. -* Components: AppShell, Nav, DataTable (virtualized), Badge/Chip, Tabs, Drawer, GraphCanvas, CodeViewer (read‑only JSON), Form primitives, Modal, Toast, Pill filters. -* Accessibility baked into components; snapshot and interaction tests in Storybook. - -> **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. - -### 5.3 Feature modules - -Each module has: - -* `routes.tsx` pages, `api` data hooks, `components`, `tests`, `docs link`. -* Query keys standardized for caching and invalidation. - -**SBOMs** - -* Hooks: `useSboms`, `useSbom(id)`, `useComponents(sbomId, query)`. -* GraphCanvas using neighborhood loaders: `/sboms/:id/graph?center=:purl&depth=1..3`. - -**Advisories** - -* `useAdvisories(filters)` and `useAdvisory(id)` plus `useLinkedAdvisories(id)`. -* UI explicitly shows aggregation groups; never collapses sources into one record. - -**VEX** - -* `useVex(filters)`, `useVexDoc(id)`, `useVexForTuple(purl, advisoryId)` for precedence views. - -**Findings** - -* `useFindings(policyId, filters, cursor)` and `useFinding(findingId)`. -* Explain viewer reading `/findings/:policyId/:findingId/explain`. - -**Policies** - -* Monaco editor wrapper; compile/simulate actions; approval dialog. -* Diff viewer using the compiler’s diagnostics and rule stats. - -**Runs** - -* `useRuns`, `useRun(runId)` + SSE hook `useRunProgress(runId)`. - -**Admin** - -* `useUsers`, `useRoles`, `useTenants`, `useTokens`, `useIntegrations`. - -**Downloads** - -* Static page with dynamic image tags fetched from registry metadata endpoint; copy‑able commands and Helm snippets. - -> **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. - -### 5.4 Live updates - -* SSE/WebSocket client with backoff, heartbeat, and re‑subscribe logic. -* Only Runs and slim “ticker” endpoints use live channels; everything else is HTTP pull with caching. - -### 5.5 Security - -* OIDC PKCE flow; token storage in memory; refresh via hidden iframe or refresh endpoint. -* CSP locked to same‑origin, with hashes for inline scripts from Next. -* Feature flags control admin features visibility; RBAC double‑checked on server responses. - -### 5.6 Packaging & distribution - -* `stella-console:` image built in CI; Nginx or Node serve. -* Helm chart values include Authority issuer, API base URL, tenant defaults. -* Offline bundle artifact for air‑gapped deployments. - ---- - -## 6) Documentation changes (create/update) - -1. **`/docs/ui/console-overview.md`** - - * Purpose, IA, tenant model, role mapping, AOC alignment. -2. **`/docs/ui/navigation.md`** - - * Route map, global filters, keyboard shortcuts, deep links. -3. **`/docs/ui/sbom-explorer.md`** - - * Catalog, detail, Graph Explorer, overlays, exports. -4. **`/docs/ui/advisories-and-vex.md`** - - * Aggregation‑not‑merge, multi‑source roll‑ups, raw viewers. -5. **`/docs/ui/findings.md`** - - * Filters, table semantics, explain view, exports. -6. **`/docs/ui/policies.md`** - - * Editor, simulation, diffs, approvals, links to DSL docs. -7. **`/docs/ui/runs.md`** - - * Queue, live progress, diffs, retries, evidence bundles. -8. **`/docs/ui/admin.md`** - - * Users, roles, tenants, tokens, integrations. -9. **`/docs/ui/downloads.md`** - - * Containers list, versions, pull/install commands, air‑gapped flow. -10. **`/docs/deploy/console.md`** - - * Helm, ingress, TLS, CSP, environment variables, health checks. -11. **`/docs/install/docker.md`** - - * All container images, pull commands, compose/Helm examples. -12. **`/docs/security/console-security.md`** - - * OIDC, RBAC, CSP, tenancy, evidence of least privilege. -13. **`/docs/observability/ui-telemetry.md`** - - * UI metrics, logs, dashboards, feature flags. -14. **`/docs/cli-vs-ui-parity.md`** - - * Matrix of operations and surfaces. -15. **`/docs/architecture/console.md`** - - * Frontend architecture, packages, data flow diagrams, SSE design. -16. **`/docs/accessibility.md`** - - * WCAG checklist, testing tools, color tokens. -17. **`/docs/examples/ui-tours.md`** - - * Task‑centric walkthroughs: triage, audit, policy rollout. - -> Each document includes a “Compliance checklist” section. -> **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. - ---- - -## 7) Tasks (tracked per team) - -### 7.1 Console scaffold & infra - -* [ ] Initialize Next.js 14 TypeScript app with App Router. -* [ ] Set up TanStack Query, Auth context, Error boundaries, Toasts. -* [ ] Integrate OIDC client; implement login/logout, tenant picker. -* [ ] Add design tokens and base components in `packages/ui`. -* [ ] Configure CI: build, test, lint, type‑check, Lighthouse budgets. -* [ ] Build `stella-console` container image and Helm chart. - -### 7.2 Typed API client - -* [ ] Generate clients from OpenAPI; wrap with hooks in `packages/api`. -* [ ] Centralize retry, error mapping, tenant header injection. - -### 7.3 Feature delivery - -**SBOMs** - -* [ ] Catalog page with filters, server pagination. -* [ ] SBOM detail with components table. -* [ ] Graph Explorer with overlays and neighborhood loaders. -* [ ] Raw JSON viewer drawer. - -**Advisories & VEX** - -* [ ] Advisory aggregation list; per‑source chips; raw view. -* [ ] VEX list with filters; precedence explainer per tuple. -* [ ] Link outs to Findings and SBOMs. - -**Findings** - -* [ ] Virtualized table; filters; saved views. -* [ ] Explain view: rules fired, references, trace links. -* [ ] Export actions (CSV/JSON stream). - -**Policies** - -* [ ] Monaco editor with syntax/diagnostics; compile and simulate. -* [ ] Diff and impact panel; submit and approve workflow. -* [ ] Run from simulation context. - -**Runs** - -* [ ] Runs list; run detail with SSE progress. -* [ ] Diff between runs; evidence bundle download. - -**Admin** - -* [ ] Users/roles CRUD; token issuance; tenant management. -* [ ] Integrations: OIDC config form; registry connections. -* [ ] Settings for environment defaults. - -**Downloads** - -* [ ] Registry tag fetch, pull commands, Helm snippet generator. -* [ ] Air‑gapped instructions and offline bundle download. - -> **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. - -### 7.4 Quality gates - -* [ ] Playwright E2E for core flows: triage, simulate, approve, run, explain. -* [ ] Storybook with a11y addon and interaction tests. -* [ ] Lighthouse CI budgets met; perf regressions block merges. -* [ ] i18n scaffolding ready; all strings externalized. -* [ ] Security checks: CSP effective, OIDC flows tested, RBAC enforced. - -### 7.5 Docs tasks - -* [ ] Populate all docs listed in section 6 with screenshots and animated GIFs. -* [ ] Add “CLI vs UI” parity matrix and keep it in CI to detect drift. -* [ ] Add “AOC user guide” callouts explaining raw fact immutability across pages. - ---- - -## 8) Feature flags - -* `ui.graph-explorer` -* `ui.policy-editor` -* `ui.ai-assist` (off by default; when enabled, renders a right‑rail for human‑in‑the‑loop summaries) -* `ui.downloads` - -Flag definitions and defaults live in `/docs/observability/ui-telemetry.md` and config map. - ---- - -## 9) Acceptance criteria - -* Console ships as a container image with Helm deployment and a static build option. -* SBOM Explorer visualizes graphs and overlays policy outcomes without page crashes on large SBOMs. -* Advisories/VEX browsers display **aggregation only**, never merge sources; raw document viewers are present. -* Findings view supports server‑side pagination and Explain with rule traces. -* Policy Editor compiles, simulates, diffs, and supports approval workflows. -* Runs page shows live progress and enables evidence exports. -* Admin handles users, roles, tenants, tokens, and OIDC configuration. -* Downloads page lists all images and installation paths. -* All pages meet a11y checks and pass Lighthouse budgets. -* RBAC enforced in UI affordances and validated by API responses. - ---- - -## 10) Risks and mitigations - -* **Graph performance** on very large SBOMs → use neighborhood windows and server filters; cap depth. -* **UI/CLI drift** → parity matrix in CI; failing check blocks release. -* **Overfetching** → TanStack caching, cursor‑based endpoints, and strict data‑layer reviews. -* **Scope creep** in Admin → feature‑flag granular sections, ship iteratively. -* **AOC confusion** → constant raw/derived labeling and “view raw” toggles. - ---- - -## 11) Test plan - -* **Unit**: hooks and components; data adapters; graph layout utils. -* **E2E**: Playwright flows for triage, simulation→approval→run→explain, admin RBAC. -* **A11y**: axe‑core in CI and manual keyboard checks. -* **Perf**: Lighthouse against seeded data; visual regression on Storybook. -* **Security**: OIDC happy and unhappy paths, CSP violation tests, SSRF resistance for downloads metadata. -* **Resilience**: simulate API timeouts; verify error boundaries and retries. - ---- - -## 12) Non‑goals (this epic) - -* No server‑side report authoring engine beyond export templates. -* No proprietary graph database; server remains RESTful with indexed queries. -* No speculative automatic policy changes; all edits remain human‑driven. - ---- - -## 13) Philosophy and guiding principles - -* **AOC first**: the UI respects facts vs decisions. Raw content is immutable and visible. -* **Deterministic outcomes**: what you see equals what the Policy Engine produced, with an explanation you can export. -* **Explainability** over cleverness: every badge, color, and status maps to a rule and a source. -* **Parity**: UI is not a second‑class citizen, and the CLI is not an afterthought. -* **Composability**: modules are independent packages with clear contracts and tests. - -> Final reminder: **Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied.** diff --git a/Mongo2Go-4.1.0.tar.gz b/Mongo2Go-4.1.0.tar.gz deleted file mode 100644 index 6f154723..00000000 Binary files a/Mongo2Go-4.1.0.tar.gz and /dev/null differ diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj b/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj index 51c40c9c..029cfb83 100644 --- a/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj @@ -1,93 +1,93 @@ - - - - net472;netstandard2.1 - Johannes Hoppe and many contributors - Mongo2Go is a managed wrapper around MongoDB binaries. It targets .NET Framework 4.7.2 and .NET Standard 2.1. -This Nuget package contains the executables of mongod, mongoimport and mongoexport v4.4.4 for Windows, Linux and macOS. - - -Mongo2Go has two use cases: - -1. Providing multiple, temporary and isolated MongoDB databases for integration tests -2. Providing a quick to set up MongoDB database for a local developer environment - HAUS HOPPE - ITS - Copyright © 2012-2025 Johannes Hoppe and many ❤️ contributors - true - icon.png - MIT - https://github.com/Mongo2Go/Mongo2Go - https://github.com/Mongo2Go/Mongo2Go/releases - MongoDB Mongo unit test integration runner - https://github.com/Mongo2Go/Mongo2Go - git - Mongo2Go - Mongo2Go is a managed wrapper around MongoDB binaries. - - - - 4 - 1701;1702;1591;1573 - - - - 4 - 1701;1702;1591;1573 - - - - 1701;1702;1591;1573 - - - - 1701;1702;1591;1573 - - - - true - true - true - - - - embedded - true - true - - - - v - - - - - - true - icon.png - - - true - tools - - - - - - - - - - - - - - - - - - - - - - - + + + + net472;netstandard2.1 + Johannes Hoppe and many contributors + Mongo2Go is a managed wrapper around MongoDB binaries. It targets .NET Framework 4.7.2 and .NET Standard 2.1. +This Nuget package contains the executables of mongod, mongoimport and mongoexport v4.4.4 for Windows, Linux and macOS. + + +Mongo2Go has two use cases: + +1. Providing multiple, temporary and isolated MongoDB databases for integration tests +2. Providing a quick to set up MongoDB database for a local developer environment + HAUS HOPPE - ITS + Copyright © 2012-2025 Johannes Hoppe and many ❤️ contributors + true + icon.png + MIT + https://github.com/Mongo2Go/Mongo2Go + https://github.com/Mongo2Go/Mongo2Go/releases + MongoDB Mongo unit test integration runner + https://github.com/Mongo2Go/Mongo2Go + git + Mongo2Go + Mongo2Go is a managed wrapper around MongoDB binaries. + + + + 4 + 1701;1702;1591;1573 + + + + 4 + 1701;1702;1591;1573 + + + + 1701;1702;1591;1573 + + + + 1701;1702;1591;1573 + + + + true + true + true + + + + embedded + true + true + + + + v + + + + + + true + icon.png + + + true + tools + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj b/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj index 7c596c21..34f3034a 100644 --- a/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj @@ -1,21 +1,21 @@ - - - net8.0 - false - - - - - - - - - - - - - - - - + + + net8.0 + false + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Mongo2Go.4.1.0.nupkg b/Mongo2Go.4.1.0.nupkg deleted file mode 100644 index a9378128..00000000 Binary files a/Mongo2Go.4.1.0.nupkg and /dev/null differ diff --git a/NuGet.config b/NuGet.config index 0578d091..225ab2b3 100644 --- a/NuGet.config +++ b/NuGet.config @@ -12,6 +12,7 @@ + diff --git a/Read SPRINTs.md b/Read SPRINTs.md deleted file mode 100644 index a075b631..00000000 --- a/Read SPRINTs.md +++ /dev/null @@ -1,7 +0,0 @@ -Read SPRINTs.md -Here follow a new EPIC you need to outline on the SPRINTS/TASKS. -Do not forget to read appropriate parts of the project to see what kind of tasks you need to edit/delete or create to forfill the goals of the sprint. Do not shy to create new projects to concentrate library or new webservice or plugin, but check the src/ for projects to make sure there is no already one that could do it. -Do not shy to edit existing tasks - if they needs to be adjusted -Do not shy to delete of existing task - if they do not make sense anymore for the new EPIC. -But most importantly create - detailed tasks to forfill the EPIC goals. - diff --git a/SPRINTS.md b/SPRINTS.md index b20be6eb..fcdce42c 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -2,121 +2,131 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | | --- | --- | --- | --- | --- | --- | --- | -| Sprint 13 | Platform Reliability | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NUGET-13-002 | Ensure all solutions/projects prioritize `local-nuget` before public feeds and add restore-order validation. | -| Sprint 13 | Platform Reliability | ops/devops/TASKS.md | TODO | DevOps Guild, Platform Leads | DEVOPS-NUGET-13-003 | Upgrade `Microsoft.*` dependencies pinned to 8.* to their latest .NET 10 (or 9.x) releases and refresh guidance. | -| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | DOING (2025-10-23) | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | -| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild, Scanner Guild | DEVOPS-REL-14-004 | Extend release/offline smoke jobs to cover Python analyzer plug-ins (warm/cold, determinism, signing). | -| Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | -| Sprint 14 | Release & Offline Ops | ops/deployment/TASKS.md | TODO | Deployment Guild | DEVOPS-OPS-14-003 | Deployment/update/rollback automation and channel management documentation. | -| Sprint 14 | Release & Offline Ops | ops/licensing/TASKS.md | TODO | Licensing Guild | DEVOPS-LIC-14-004 | Registry token service tied to Authority, plan gating, revocation handling, monitoring. | -| Sprint 16 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-EVENTS-16-301 | Rework scanner event publication/tests to emit `ORCH-SVC-38-101` envelopes for Notifier ingestion (no Redis dependency). | -| Sprint 15 | Benchmarks | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild, Notify Team | BENCH-NOTIFY-15-001 | Notify dispatch throughput bench with results CSV. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-101 | Define Scheduler DTOs & validation. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-102 | Publish schema docs/sample payloads. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Mongo schemas/indexes for Scheduler state. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-202 | Repositories with tenant scoping, TTL, causal consistency. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-203 | Audit + stats materialization for UI. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Ingest BOM-Index into roaring bitmap store. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-302 | Query APIs for ResolveByPurls/ResolveByVulns/ResolveAll. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-303 | Snapshot/compaction/invalidation workflow. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Minimal API host with Authority enforcement. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Schedules CRUD (cron validation, pause/resume, audit). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-103 | Runs API (list/detail/cancel) + impact previews. | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-104 | Feedser/Vexer webhook handlers with security enforcement. | +| Sprint 13 | Platform Reliability | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-NUGET-13-002 | Ensure all solutions/projects prioritize `local-nuget` before public feeds and add restore-order validation. | +| Sprint 13 | Platform Reliability | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild, Platform Leads | DEVOPS-NUGET-13-003 | Upgrade `Microsoft.*` dependencies pinned to 8.* to their latest .NET 10 (or 9.x) releases and refresh guidance. | +| Sprint 14 | Release & Offline Ops | ops/deployment/TASKS.md | DONE (2025-10-26) | Deployment Guild | DEVOPS-OPS-14-003 | Deployment/update/rollback automation and channel management documentation. | +| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | +| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild, Scanner Guild | DEVOPS-REL-14-004 | Extend release/offline smoke jobs to cover Python analyzer plug-ins (warm/cold, determinism, signing). | +| Sprint 14 | Release & Offline Ops | ops/licensing/TASKS.md | DONE (2025-10-26) | Licensing Guild | DEVOPS-LIC-14-004 | Registry token service tied to Authority, plan gating, revocation handling, monitoring. | +| Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | DONE (2025-10-26) | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | +| Sprint 15 | Benchmarks | src/StellaOps.Bench/TASKS.md | DONE (2025-10-26) | Bench Guild, Notify Team | BENCH-NOTIFY-15-001 | Notify dispatch throughput bench with results CSV. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Define Scheduler DTOs & validation. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-102 | Publish schema docs/sample payloads. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Mongo schemas/indexes for Scheduler state. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-202 | Repositories with tenant scoping, TTL, causal consistency. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-203 | Audit/run stats materialization for UI. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-302 | Query APIs for ResolveByPurls/ResolveByVulns/ResolveAll. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Ingest BOM-Index into roaring bitmap store. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | DOING (2025-10-19) | Scheduler WebService Guild | SCHED-WEB-16-101 | Minimal API host with Authority enforcement. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-102 | Schedules CRUD (cron validation, pause/resume, audit). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-103 | Runs API (list/detail/cancel) + impact previews. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | DONE (2025-10-27) | Scheduler WebService Guild | SCHED-WEB-16-104 | Feedser/Vexer webhook handlers with security enforcement. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-201 | Planner loop (cron/event triggers, leases, fairness). | -| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-202 | ImpactIndex targeting and shard planning. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | DOING (2025-10-26) | Scheduler Worker Guild | SCHED-WORKER-16-202 | ImpactIndex targeting and shard planning. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-203 | Runner execution invoking Scanner analysis/content refresh. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-204 | Emit rescan/report events for Notify/UI. | | Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-205 | Metrics/telemetry for Scheduler planners/runners. | -| Sprint 16 | Benchmarks | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild, Scheduler Team | BENCH-IMPACT-16-001 | ImpactIndex throughput bench + RAM profile. | -| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-17-701 | Record GNU build-id for ELF components and surface it in SBOM/diff outputs. | -| Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | -| Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | -| Sprint 18 | Launch Readiness | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LAUNCH-18-001 | Production launch cutover rehearsal and runbook publication (blocked on implementation sign-off and environment setup). | -| Sprint 18 | Launch Readiness | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild, Scanner Guild | DEVOPS-OFFLINE-18-005 | Rebuild Offline Kit with Python analyzer artefacts and refreshed manifest/signature pair. | -| Sprint 35 | EPDR Foundations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild | SCANNER-ANALYZERS-LANG-11-001 | Build entrypoint resolver (identity + environment profiles) and emit normalized entrypoint records. | -| Sprint 35 | EPDR Foundations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild | SCANNER-ANALYZERS-LANG-11-002 | Static IL/reflection/ALC heuristics producing dependency edges with reason codes and confidence. | -| Sprint 35 | EPDR Foundations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild, Signals Guild | SCANNER-ANALYZERS-LANG-11-003 | Runtime loader/PInvoke signal ingestion merged with static/declared edges (confidence & explain). | -| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-001 | Format detector & binary identity for ELF/PE/Mach-O (multi-slice) with stable entrypoint IDs. | -| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-002 | ELF dynamic parser emitting dtneeded edges, runpath metadata, symbol version needs. | -| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-003 | PE import + delay-load + SxS manifest parsing producing reason-coded edges. | -| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-004 | Mach-O load command parsing with @rpath expansion and slice handling. | -| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-005 | Cross-platform resolver engine modeling search order/explain traces for ELF/PE/Mach-O. | -| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-006 | Heuristic scanner for dlopen/LoadLibrary strings, plugin configs, ecosystem hints with confidence tags. | -| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-007 | Serialize entrypoints/edges/env profiles to Scanner writer (AOC-compliant observations). | -| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild, QA Guild | SCANNER-ANALYZERS-NATIVE-20-008 | Fixture suite + determinism benchmarks for native analyzer across linux/windows/macos. | -| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-NATIVE-20-009 | Optional runtime capture adapters (eBPF/ETW/dyld) producing runtime-load edges with redaction. | -| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-NATIVE-20-010 | Package native analyzer plug-in + Offline Kit updates and restart-time loading. | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-001 | Java input normalizer (jar/war/ear/fat/jmod/jimage) with MR overlay selection. | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | Module/classpath builder with duplicate & split-package detection. | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-003 | SPI scanner & provider selection with warnings. | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-004 | Reflection/TCCL heuristics emitting reason-coded edges. | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-005 | Framework config extraction (Spring, Jakarta, MicroProfile, logging, Graal configs). | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-006 | JNI/native hint detection for Java artifacts. | -| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-007 | Manifest/signature metadata collector (main/start/agent classes, signers). | -| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-008 | Observation writer producing entrypoints/components/edges with warnings. | -| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild, QA Guild | SCANNER-ANALYZERS-JAVA-21-009 | Fixture suite + determinism/perf benchmarks for Java analyzer. | -| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-JAVA-21-010 | Optional runtime ingestion via agent/JFR producing runtime edges. | -| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-JAVA-21-011 | Package Java analyzer plug-in + Offline Kit/CLI updates. | -| Sprint 36 | EPDR Observations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild, SBOM Service Guild | SCANNER-ANALYZERS-LANG-11-004 | Normalize EPDR output to Scanner observation writer (entrypoints + edges + env profiles). | -| Sprint 36 | EPDR Observations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild, QA Guild | SCANNER-ANALYZERS-LANG-11-005 | End-to-end fixtures/benchmarks covering publish modes, RIDs, trimming, NativeAOT with explain traces. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-AOC-19-001 | Implement raw advisory ingestion endpoints with AOC guard and verifier. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-WEB-AOC-19-002 | Emit AOC observability metrics, traces, and structured logs. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | QA Guild | CONCELIER-WEB-AOC-19-003 | Add schema/guard unit tests covering AOC error codes. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-004 | Build integration suite validating deterministic ingest under load. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-001 | Implement AOC repository guard rejecting forbidden fields. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-002 | Deliver deterministic linkset extraction for advisories. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-003 | Enforce idempotent append-only upsert with supersedes pointers. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-004 | Remove ingestion normalization; defer derived logic to Policy Engine. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-001 | Add Mongo schema validator for `advisory_raw`. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-002 | Create idempotency unique index backed by migration scripts. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-003 | Deliver append-only migration/backfill plan with supersedes chaining. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild, DevOps Guild | CONCELIER-STORE-AOC-19-004 | Document validator deployment steps for online/offline clusters. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-AOC-19-001 | Implement raw VEX ingestion and AOC verifier endpoints. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-002 | Emit AOC metrics/traces/logging for Excititor ingestion. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | QA Guild | EXCITITOR-WEB-AOC-19-003 | Add AOC guard test harness for VEX schemas. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-004 | Validate large VEX ingest runs and CLI verification parity. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-001 | Introduce VEX repository guard enforcing AOC invariants. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-002 | Build deterministic VEX linkset extraction. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-003 | Enforce append-only idempotent VEX raw upserts. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-004 | Remove ingestion consensus logic; rely on Policy Engine. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-STORE-AOC-19-001 | Add Mongo schema validator for `vex_raw`. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-STORE-AOC-19-002 | Create idempotency unique index for VEX raw documents. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-STORE-AOC-19-003 | Deliver append-only migration/backfill for VEX raw collections. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild, DevOps Guild | EXCITITOR-STORE-AOC-19-004 | Document validator deployment for Excititor clusters/offline kit. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-WORKER-AOC-19-001 | Rewire worker to persist raw VEX docs with guard enforcement. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-WORKER-AOC-19-002 | Enforce signature/checksum verification prior to raw writes. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Worker/TASKS.md | TODO | QA Guild | EXCITITOR-WORKER-AOC-19-003 | Expand worker tests for deterministic batching and restart safety. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AOC-19-001 | Provide shared AOC forbidden key set and guard middleware. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AOC-19-002 | Ship provenance builder and signature helpers for ingestion services. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, QA Guild | WEB-AOC-19-003 | Author analyzer + shared test fixtures for guard compliance. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-AOC-19-001 | Add lint preventing ingestion modules from referencing Policy-only helpers. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild, Security Guild | POLICY-AOC-19-002 | Enforce Policy-only writes to `effective_finding_*` collections. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-AOC-19-003 | Update Policy readers to consume only raw document fields. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild, QA Guild | POLICY-AOC-19-004 | Add determinism tests for raw-driven policy recomputation. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-AOC-19-001 | Introduce new ingestion/auth scopes across Authority. | -| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-AOC-19-002 | Enforce tenant claim propagation and cross-tenant guardrails. | +| Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | +| Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | +| Sprint 17 | Symbol Intelligence & Forensics | ops/offline-kit/TASKS.md | DONE (2025-10-26) | Offline Kit Guild, DevOps Guild | DEVOPS-OFFLINE-17-003 | Mirror release debug-store artefacts into Offline Kit packaging and document validation. | +| Sprint 17 | Symbol Intelligence & Forensics | ops/offline-kit/TASKS.md | BLOCKED (2025-10-26) | Offline Kit Guild, DevOps Guild | DEVOPS-OFFLINE-17-004 | Run mirror_debug_store.py once release artefacts exist and archive verification evidence with the Offline Kit. | +| Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | BLOCKED (2025-10-26) | DevOps Guild | DEVOPS-REL-17-004 | Ensure release workflow publishes `out/release/debug` (build-id tree + manifest) and fails when symbols are missing. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.Emit/TASKS.md | DONE (2025-10-26) | Emit Guild | SCANNER-EMIT-17-701 | Record GNU build-id for ELF components and surface it in SBOM/diff outputs. | +| Sprint 18 | Launch Readiness | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-LAUNCH-18-001 | Production launch cutover rehearsal and runbook publication. | +| Sprint 18 | Launch Readiness | ops/offline-kit/TASKS.md | DONE (2025-10-26) | Offline Kit Guild, Scanner Guild | DEVOPS-OFFLINE-18-005 | Rebuild Offline Kit with Python analyzer artefacts and refreshed manifest/signature pair. | +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-AOC-19-001 | Publish aggregation-only contract reference documentation. | +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Architecture Guild | DOCS-AOC-19-002 | Update architecture overview with AOC boundary diagrams. | +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Policy Guild | DOCS-AOC-19-003 | Refresh policy engine doc with raw ingestion constraints. | +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, UI Guild | DOCS-AOC-19-004 | Document console AOC dashboard and drill-down flow. | +> DOCS-AOC-19-004: Architecture overview & policy-engine docs refreshed 2025-10-26 — reuse new AOC boundary diagram + metrics guidance. +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, CLI Guild | DOCS-AOC-19-005 | Document CLI AOC commands and exit codes. | +> DOCS-AOC-19-005: Link to the new AOC reference and architecture overview; include exit code table sourced from those docs. +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Observability Guild | DOCS-AOC-19-006 | Document new AOC metrics, traces, and logs. | +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Authority Core | DOCS-AOC-19-007 | Document new Authority scopes and tenancy enforcement. | +| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, DevOps Guild | DOCS-AOC-19-008 | Update deployment guide with validator enablement and verify user guidance. | +| Sprint 19 | Aggregation-Only Contract Enforcement | ops/devops/TASKS.md | BLOCKED (2025-10-26) | DevOps Guild, Platform Guild | DEVOPS-AOC-19-001 | Integrate AOC analyzer/guard enforcement into CI pipelines. | +| Sprint 19 | Aggregation-Only Contract Enforcement | ops/devops/TASKS.md | BLOCKED (2025-10-26) | DevOps Guild | DEVOPS-AOC-19-002 | Add CI stage running `stella aoc verify` against seeded snapshots. | +| Sprint 19 | Aggregation-Only Contract Enforcement | ops/devops/TASKS.md | BLOCKED (2025-10-26) | DevOps Guild, QA Guild | DEVOPS-AOC-19-003 | Enforce guard coverage thresholds and export metrics to dashboards. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-AOC-19-001 | Introduce new ingestion/auth scopes across Authority. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Authority/TASKS.md | DOING (2025-10-26) | Authority Core & Security Guild | AUTH-AOC-19-002 | Enforce tenant claim propagation and cross-tenant guardrails. | +> AUTH-AOC-19-002: Tenant metadata now flows through rate limiter/audit/token persistence; password grant scope/tenant enforcement landed. Docs/stakeholder walkthrough pending. | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-AOC-19-003 | Update Authority docs/config samples for new scopes. | | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AOC-19-001 | Implement `stella sources ingest --dry-run` command. | | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AOC-19-002 | Implement `stella aoc verify` command with exit codes. | | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Cli/TASKS.md | TODO | Docs/CLI Guild | CLI-AOC-19-003 | Update CLI reference and quickstart docs for new AOC commands. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-001 | Implement AOC repository guard rejecting forbidden fields. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-002 | Deliver deterministic linkset extraction for advisories. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-003 | Enforce idempotent append-only upsert with supersedes pointers. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-004 | Remove ingestion normalization; defer derived logic to Policy Engine. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-013 | Extend smoke coverage to validate tenant-scoped Authority tokens and cross-tenant rejection. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-001 | Add Mongo schema validator for `advisory_raw`. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-002 | Create idempotency unique index backed by migration scripts. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-003 | Deliver append-only migration/backfill plan with supersedes chaining. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild, DevOps Guild | CONCELIER-STORE-AOC-19-004 | Document validator deployment steps for online/offline clusters. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-AOC-19-001 | Implement raw advisory ingestion endpoints with AOC guard and verifier. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-WEB-AOC-19-002 | Emit AOC observability metrics, traces, and structured logs. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | QA Guild | CONCELIER-WEB-AOC-19-003 | Add schema/guard unit tests covering AOC error codes. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-004 | Build integration suite validating deterministic ingest under load. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-001 | Introduce VEX repository guard enforcing AOC invariants. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-002 | Build deterministic VEX linkset extraction. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-003 | Enforce append-only idempotent VEX raw upserts. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-004 | Remove ingestion consensus logic; rely on Policy Engine. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-013 | Update smoke suites to enforce tenant-scoped Authority tokens and cross-tenant VEX rejection. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-STORE-AOC-19-001 | Add Mongo schema validator for `vex_raw`. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-STORE-AOC-19-002 | Create idempotency unique index for VEX raw documents. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-STORE-AOC-19-003 | Deliver append-only migration/backfill for VEX raw collections. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild, DevOps Guild | EXCITITOR-STORE-AOC-19-004 | Document validator deployment for Excititor clusters/offline kit. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-AOC-19-001 | Implement raw VEX ingestion and AOC verifier endpoints. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-002 | Emit AOC metrics/traces/logging for Excititor ingestion. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | QA Guild | EXCITITOR-WEB-AOC-19-003 | Add AOC guard test harness for VEX schemas. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-004 | Validate large VEX ingest runs and CLI verification parity. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-WORKER-AOC-19-001 | Rewire worker to persist raw VEX docs with guard enforcement. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-WORKER-AOC-19-002 | Enforce signature/checksum verification prior to raw writes. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Excititor.Worker/TASKS.md | TODO | QA Guild | EXCITITOR-WORKER-AOC-19-003 | Expand worker tests for deterministic batching and restart safety. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-AOC-19-001 | Add lint preventing ingestion modules from referencing Policy-only helpers. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild, Security Guild | POLICY-AOC-19-002 | Enforce Policy-only writes to `effective_finding_*` collections. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-AOC-19-003 | Update Policy readers to consume only raw document fields. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild, QA Guild | POLICY-AOC-19-004 | Add determinism tests for raw-driven policy recomputation. | | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AOC-19-001 | Add Sources dashboard tiles surfacing AOC status and violations. | | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AOC-19-002 | Build violation drill-down view for offending documents. | | Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AOC-19-003 | Wire "Verify last 24h" action and CLI parity messaging. | -| Sprint 19 | Aggregation-Only Contract Enforcement | ops/devops/TASKS.md | TODO | DevOps Guild, Platform Guild | DEVOPS-AOC-19-001 | Integrate AOC analyzer/guard enforcement into CI pipelines. | -| Sprint 19 | Aggregation-Only Contract Enforcement | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AOC-19-002 | Add CI stage running `stella aoc verify` against seeded snapshots. | -| Sprint 19 | Aggregation-Only Contract Enforcement | ops/devops/TASKS.md | TODO | DevOps Guild, QA Guild | DEVOPS-AOC-19-003 | Enforce guard coverage thresholds and export metrics to dashboards. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild | DOCS-AOC-19-001 | Publish aggregation-only contract reference documentation. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, Architecture Guild | DOCS-AOC-19-002 | Update architecture overview with AOC boundary diagrams. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, Policy Guild | DOCS-AOC-19-003 | Refresh policy engine doc with raw ingestion constraints. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, UI Guild | DOCS-AOC-19-004 | Document console AOC dashboard and drill-down flow. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, CLI Guild | DOCS-AOC-19-005 | Document CLI AOC commands and exit codes. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, Observability Guild | DOCS-AOC-19-006 | Document new AOC metrics, traces, and logs. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, Authority Core | DOCS-AOC-19-007 | Document new Authority scopes and tenancy enforcement. | -| Sprint 19 | Aggregation-Only Contract Enforcement | docs/TASKS.md | TODO | Docs Guild, DevOps Guild | DOCS-AOC-19-008 | Update deployment guide with validator enablement and verify user guidance. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Platform Guild | POLICY-ENGINE-20-000 | Spin up new Policy Engine service host with DI bootstrap and Authority wiring. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-20-001 | Deliver `stella-dsl@1` parser + IR compiler with diagnostics and checksums. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-20-002 | Implement deterministic rule evaluator with priority/first-match semantics. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Web/TASKS.md | DOING (2025-10-26) | BE-Base Platform Guild | WEB-AOC-19-001 | Provide shared AOC forbidden key set and guard middleware. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AOC-19-002 | Ship provenance builder and signature helpers for ingestion services. | +| Sprint 19 | Aggregation-Only Contract Enforcement | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, QA Guild | WEB-AOC-19-003 | Author analyzer + shared test fixtures for guard compliance. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-POLICY-20-001 | Publish `/docs/policy/overview.md` with compliance checklist. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-POLICY-20-002 | Document DSL grammar + examples in `/docs/policy/dsl.md`. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Authority Core | DOCS-POLICY-20-003 | Write `/docs/policy/lifecycle.md` covering workflow + roles. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Scheduler Guild | DOCS-POLICY-20-004 | Document policy run modes + cursors in `/docs/policy/runs.md`. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Platform Guild | DOCS-POLICY-20-005 | Produce `/docs/api/policy.md` with endpoint schemas + errors. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, CLI Guild | DOCS-POLICY-20-006 | Author `/docs/cli/policy.md` with commands, exit codes, JSON output. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, UI Guild | DOCS-POLICY-20-007 | Create `/docs/ui/policy-editor.md` covering editor, simulation, approvals. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Architecture Guild | DOCS-POLICY-20-008 | Publish `/docs/architecture/policy-engine.md` with sequence diagrams. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Observability Guild | DOCS-POLICY-20-009 | Document metrics/traces/logs in `/docs/observability/policy.md`. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Security Guild | DOCS-POLICY-20-010 | Publish `/docs/security/policy-governance.md` for scopes + approvals. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Policy Guild | DOCS-POLICY-20-011 | Add example policies under `/docs/examples/policies/` with commentary. | +| Sprint 20 | Policy Engine v2 | docs/TASKS.md | DONE (2025-10-26) | Docs Guild, Support Guild | DOCS-POLICY-20-012 | Draft `/docs/faq/policy-faq.md` covering conflicts, determinism, pitfalls. | +| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-POLICY-20-001 | Add DSL lint + compile checks to CI pipelines. | +| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | BLOCKED (waiting on POLICY-ENGINE-20-006) | DevOps Guild | DEVOPS-POLICY-20-002 | Run `stella policy simulate` CI stage against golden SBOMs. | +| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild, QA Guild | DEVOPS-POLICY-20-003 | Add determinism CI job diffing repeated policy runs. | +| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | DOING (2025-10-26) | DevOps Guild, Scheduler Guild, CLI Guild | DEVOPS-POLICY-20-004 | Automate policy schema exports and change notifications for CLI consumers. | +| Sprint 20 | Policy Engine v2 | samples/TASKS.md | DONE (2025-10-26) | Samples Guild, Policy Guild | SAMPLES-POLICY-20-001 | Commit baseline/serverless/internal-only policy samples + fixtures. | +| Sprint 20 | Policy Engine v2 | samples/TASKS.md | DONE (2025-10-26) | Samples Guild, UI Guild | SAMPLES-POLICY-20-002 | Produce simulation diff fixtures for UI/CLI tests. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-POLICY-20-001 | Add new policy scopes (`policy:*`, `findings:read`, `effective:write`). | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-POLICY-20-002 | Enforce Policy Engine service identity and scope checks at gateway. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core & Docs Guild | AUTH-POLICY-20-003 | Update Authority docs/config samples for policy scopes + workflows. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Bench/TASKS.md | DONE (2025-10-26) | Bench Guild, Policy Guild | BENCH-POLICY-20-001 | Create policy evaluation benchmark suite + baseline metrics. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Bench/TASKS.md | BLOCKED (waiting on SCHED-WORKER-20-302) | Bench Guild, Scheduler Guild | BENCH-POLICY-20-002 | Add incremental run benchmark capturing delta SLA compliance. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-20-002 | Implement `stella policy simulate` with diff outputs + exit codes. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild, Docs Guild | CLI-POLICY-20-003 | Extend `stella findings` commands with policy filters and explain view. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-POLICY-20-002 | Strengthen linkset builders with equivalence tables + range parsing. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-POLICY-20-003 | Add advisory selection cursors + change-stream checkpoints for policy runs. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-POLICY-20-001 | Provide advisory selection endpoints for policy engine (batch PURL/ID). | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-POLICY-20-002 | Enhance VEX linkset scope + version resolution for policy accuracy. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-POLICY-20-003 | Introduce VEX selection cursors + change-stream checkpoints. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-POLICY-20-001 | Ship VEX selection APIs aligned with policy join requirements. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | DONE (2025-10-26) | Policy Guild, Platform Guild | POLICY-ENGINE-20-000 | Spin up new Policy Engine service host with DI bootstrap and Authority wiring. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | DONE (2025-10-26) | Policy Guild | POLICY-ENGINE-20-001 | Deliver `stella-dsl@1` parser + IR compiler with diagnostics and checksums. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | BLOCKED (2025-10-26) | Policy Guild | POLICY-ENGINE-20-002 | Implement deterministic rule evaluator with priority/first-match semantics. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Concelier Core, Excititor Core | POLICY-ENGINE-20-003 | Build SBOM↔advisory↔VEX linkset joiners with deterministic batching. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Storage Guild | POLICY-ENGINE-20-004 | Materialize effective findings with append-only history and tenant scoping. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Security Guild | POLICY-ENGINE-20-005 | Enforce determinism guard banning wall-clock, RNG, and network usage. | @@ -124,123 +134,62 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Observability Guild | POLICY-ENGINE-20-007 | Emit policy metrics, traces, and sampled rule-hit logs. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, QA Guild | POLICY-ENGINE-20-008 | Add unit/property/golden/perf suites verifying determinism + SLA. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Storage Guild | POLICY-ENGINE-20-009 | Define Mongo schemas/indexes + migrations for policies/runs/findings. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-20-001 | Implement Policy CRUD/compile/run/simulate/findings/explain endpoints. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-20-002 | Add pagination, filters, deterministic ordering to policy listings. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, QA Guild | WEB-POLICY-20-003 | Map engine errors to `ERR_POL_*` responses with contract tests. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | Platform Reliability Guild | WEB-POLICY-20-004 | Introduce rate limits/quotas + metrics for simulation endpoints. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-POLICY-20-001 | Ship Monaco-based policy editor with inline diagnostics + checklists. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-POLICY-20-002 | Build simulation panel with deterministic diff rendering + virtualization. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild, Product Ops | UI-POLICY-20-003 | Implement submit/review/approve workflow with RBAC + audit trail. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild, Observability Guild | UI-POLICY-20-004 | Add run dashboards (heatmap/VEX wins/suppressions) with export. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-20-001 | Add `stella policy new|edit|submit|approve` commands with approvals flow. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-20-002 | Implement `stella policy simulate` with diff outputs + exit codes. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild, Docs Guild | CLI-POLICY-20-003 | Extend `stella findings` commands with policy filters and explain view. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-POLICY-20-001 | Provide advisory selection endpoints for policy engine (batch PURL/ID). | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-POLICY-20-002 | Strengthen linkset builders with equivalence tables + range parsing. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-POLICY-20-003 | Add advisory selection cursors + change-stream checkpoints for policy runs. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-POLICY-20-001 | Ship VEX selection APIs aligned with policy join requirements. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-POLICY-20-002 | Enhance VEX linkset scope + version resolution for policy accuracy. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-POLICY-20-003 | Introduce VEX selection cursors + change-stream checkpoints. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-20-001 | Define policy run/diff DTOs + validation helpers. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-26) | Scheduler Models Guild | SCHED-MODELS-20-001 | Define policy run/diff DTOs + validation helpers. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-20-002 | Update schema docs with policy run lifecycle samples. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-20-001 | Expose policy run scheduling APIs with scope enforcement. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-20-002 | Provide simulation trigger endpoint returning diff metadata. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-20-301 | Schedule policy runs via API with idempotent job tracking. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-20-302 | Implement delta targeting leveraging change streams + policy metadata. | | Sprint 20 | Policy Engine v2 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild, Observability Guild | SCHED-WORKER-20-303 | Expose policy scheduling metrics/logs with policy/run identifiers. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-POLICY-20-001 | Add new policy scopes (`policy:*`, `findings:read`, `effective:write`). | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-POLICY-20-002 | Enforce Policy Engine service identity and scope checks at gateway. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-POLICY-20-003 | Update Authority docs/config samples for policy scopes + workflows. | -| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-POLICY-20-001 | Add DSL lint + compile checks to CI pipelines. | -| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-POLICY-20-002 | Run `stella policy simulate` CI stage against golden SBOMs. | -| Sprint 20 | Policy Engine v2 | ops/devops/TASKS.md | TODO | DevOps Guild, QA Guild | DEVOPS-POLICY-20-003 | Add determinism CI job diffing repeated policy runs. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild | DOCS-POLICY-20-001 | Publish `/docs/policy/overview.md` with compliance checklist. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild | DOCS-POLICY-20-002 | Document DSL grammar + examples in `/docs/policy/dsl.md`. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Authority Core | DOCS-POLICY-20-003 | Write `/docs/policy/lifecycle.md` covering workflow + roles. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Scheduler Guild | DOCS-POLICY-20-004 | Document policy run modes + cursors in `/docs/policy/runs.md`. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Platform Guild | DOCS-POLICY-20-005 | Produce `/docs/api/policy.md` with endpoint schemas + errors. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, CLI Guild | DOCS-POLICY-20-006 | Author `/docs/cli/policy.md` with commands, exit codes, JSON output. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, UI Guild | DOCS-POLICY-20-007 | Create `/docs/ui/policy-editor.md` covering editor, simulation, approvals. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Architecture Guild | DOCS-POLICY-20-008 | Publish `/docs/architecture/policy-engine.md` with sequence diagrams. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Observability Guild | DOCS-POLICY-20-009 | Document metrics/traces/logs in `/docs/observability/policy.md`. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Security Guild | DOCS-POLICY-20-010 | Publish `/docs/security/policy-governance.md` for scopes + approvals. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Policy Guild | DOCS-POLICY-20-011 | Add example policies under `/docs/examples/policies/` with commentary. | -| Sprint 20 | Policy Engine v2 | docs/TASKS.md | TODO | Docs Guild, Support Guild | DOCS-POLICY-20-012 | Draft `/docs/faq/policy-faq.md` covering conflicts, determinism, pitfalls. | -| Sprint 20 | Policy Engine v2 | samples/TASKS.md | TODO | Samples Guild, Policy Guild | SAMPLES-POLICY-20-001 | Commit baseline/serverless/internal-only policy samples + fixtures. | -| Sprint 20 | Policy Engine v2 | samples/TASKS.md | TODO | Samples Guild, UI Guild | SAMPLES-POLICY-20-002 | Produce simulation diff fixtures for UI/CLI tests. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild, Policy Guild | BENCH-POLICY-20-001 | Create policy evaluation benchmark suite + baseline metrics. | -| Sprint 20 | Policy Engine v2 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild, Scheduler Guild | BENCH-POLICY-20-002 | Add incremental run benchmark capturing delta SLA compliance. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer Guild | CARTO-GRAPH-21-001 | Define graph storage schema, sharding strategy, and indexes for snapshots/nodes/edges/overlays. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer Guild | CARTO-GRAPH-21-002 | Implement SBOM projection reader consuming normalized CycloneDX/SPDX with entrypoint tagging. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer Guild | CARTO-GRAPH-21-003 | Build graph constructor with PURL dedupe and metadata enrichment. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer Guild | CARTO-GRAPH-21-004 | Implement layout/tiling pipeline and persist tiles to object storage. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer Guild, Policy Guild | CARTO-GRAPH-21-005 | Overlay worker hydrating policy findings and computing path relevance. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer & BE-Base Guilds | CARTO-GRAPH-21-006 | Expose graph APIs (versions, viewport, node, paths, diff, export, simulate) with streaming. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer & Scheduler Guilds | CARTO-GRAPH-21-007 | Build backfill + incremental overlay jobs using change streams and scheduler hooks. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer & QA Guilds | CARTO-GRAPH-21-008 | Deliver test/perf suites and determinism checks for large graphs. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer & DevOps Guilds | CARTO-GRAPH-21-009 | Provide deployment artefacts and offline kit guidance for Cartographer. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-POLICY-20-001 | Ship Monaco-based policy editor with inline diagnostics + checklists. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-POLICY-20-002 | Build simulation panel with deterministic diff rendering + virtualization. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild, Product Ops | UI-POLICY-20-003 | Implement submit/review/approve workflow with RBAC + audit trail. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.UI/TASKS.md | TODO | UI Guild, Observability Guild | UI-POLICY-20-004 | Add run dashboards (heatmap/VEX wins/suppressions) with export. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-20-001 | Implement Policy CRUD/compile/run/simulate/findings/explain endpoints. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-20-002 | Add pagination, filters, deterministic ordering to policy listings. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, QA Guild | WEB-POLICY-20-003 | Map engine errors to `ERR_POL_*` responses with contract tests. | +| Sprint 20 | Policy Engine v2 | src/StellaOps.Web/TASKS.md | TODO | Platform Reliability Guild | WEB-POLICY-20-004 | Introduce rate limits/quotas + metrics for simulation endpoints. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core Guild | AUTH-GRAPH-21-001 | Introduce graph scopes (`graph:*`) with configuration binding and defaults. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core Guild | AUTH-GRAPH-21-002 | Enforce graph scopes/identities at gateway with tenant propagation. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Authority/TASKS.md | DONE (2025-10-26) | Authority Core & Docs Guild | AUTH-GRAPH-21-003 | Update security docs/config samples for graph access and least privilege. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild, Graph Platform Guild | BENCH-GRAPH-21-001 | Graph viewport/path perf harness (50k/100k nodes) measuring Graph API/Indexer latency and cache hit rates. Executed within Sprint 28 Graph program. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild, UI Guild | BENCH-GRAPH-21-002 | Headless UI load benchmark for graph canvas interactions (Playwright) tracking render FPS budgets. Executed within Sprint 28 Graph program. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-GRAPH-21-001 | Enrich SBOM normalization with relationships, scopes, entrypoint annotations for Cartographer. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core & Scheduler Guilds | CONCELIER-GRAPH-21-002 | Publish SBOM change events with tenant metadata for graph builds. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cartographer/TASKS.md | TODO | Cartographer Guild | CARTO-GRAPH-21-010 | Replace hard-coded `graph:*` scope strings with shared constants once graph services integrate. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-GRAPH-21-001 | Deliver batched VEX/advisory fetch helpers for inspector linkouts. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-GRAPH-21-002 | Enrich overlay metadata with VEX justification summaries for graph overlays. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-GRAPH-21-005 | Create indexes/materialized views for VEX lookups by PURL/policy. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-SERVICE-21-001 | Expose normalized SBOM projection API with relationships, scopes, entrypoints. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service & Scheduler Guilds | SBOM-SERVICE-21-002 | Emit SBOM version change events for Cartographer build queue. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-SERVICE-21-003 | Provide entrypoint management API with tenant overrides. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service & Observability Guilds | SBOM-SERVICE-21-004 | Add metrics/traces/logs for SBOM projections. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-GRAPH-21-001 | Enrich SBOM normalization with relationships, scopes, entrypoint annotations for Cartographer. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core & Scheduler Guilds | CONCELIER-GRAPH-21-002 | Publish SBOM change events with tenant metadata for graph builds. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-GRAPH-21-003 | Expose SBOM projection endpoint for Cartographer consumption. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-GRAPH-21-004 | Provide entrypoint lookup API supporting overrides and defaults. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-GRAPH-21-001 | Deliver batched VEX/advisory fetch helpers for inspector linkouts. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-GRAPH-21-002 | Enrich overlay metadata with VEX justification summaries for graph overlays. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-21-003 | Add API endpoints returning VEX statements for inspector panels. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-21-004 | Emit events on new VEX docs for Cartographer overlay refresh. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-GRAPH-21-005 | Create indexes/materialized views for VEX lookups by PURL/policy. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy & Cartographer Guilds | POLICY-ENGINE-30-001 | Define graph overlay contract and projection API for nodes/edges. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy & Cartographer Guilds | POLICY-ENGINE-30-002 | Implement simulation overlay bridge for Cartographer and Graph Explorer. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy & Scheduler Guilds | POLICY-ENGINE-30-003 | Emit effective finding change events tailored for graph overlay refresh. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-26) | Scheduler Models Guild | SCHED-MODELS-21-001 | Define job DTOs for graph builds/overlay refresh (`GraphBuildJob`, `GraphOverlayJob`) with deterministic serialization and status enums; document in `src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md`. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-26) | Scheduler Models Guild | SCHED-MODELS-21-002 | Publish schema docs/sample payloads for graph job lifecycle. | +| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-21-002 | Expose overlay lag metrics and job completion hooks for Cartographer. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-21-001 | Add gateway routes for graph APIs with scope enforcement and streaming. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-21-002 | Implement bbox/zoom/path validation and pagination for graph endpoints. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & QA Guilds | WEB-GRAPH-21-003 | Map graph errors to `ERR_Graph_*` and support export streaming. | | Sprint 21 | Graph Explorer v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base & Policy Guilds | WEB-GRAPH-21-004 | Wire Policy Engine simulation overlays into graph responses. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-GRAPH-21-001 | Build virtualized graph canvas with clustering and severity overlays. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-GRAPH-21-002 | Deliver inspector panel with metadata, findings, VEX rationale, copy/export. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-GRAPH-21-003 | Implement filter/search experience with debounced API calls and permalinks. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI & Policy Guilds | UI-GRAPH-21-004 | Add path view and simulation overlay toggle with exports. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-GRAPH-21-005 | Provide time-travel + diff visualization between SBOM versions. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI & Accessibility Guilds | UI-GRAPH-21-006 | Ship accessibility features, keyboard nav, high-contrast mode, permalinks. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-21-001 | Implement `stella sbom graph build/export/query/diff` commands. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-21-002 | Add path query & simulation options with JSON output. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI & Docs Guilds | CLI-GRAPH-21-003 | Document CLI usage and provide fixtures for CI smoke tests. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-GRAPH-21-001 | Introduce graph scopes (`graph:*`) with configuration binding and defaults. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-GRAPH-21-002 | Enforce graph scopes/identities at gateway with tenant propagation. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-GRAPH-21-003 | Update security docs/config samples for graph access and least privilege. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-21-001 | Define job DTOs for graph builds/overlay refresh with deterministic serialization. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-21-002 | Publish schema docs/sample payloads for graph job lifecycle. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-21-001 | Add APIs to schedule/monitor graph build & overlay jobs with RBAC. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-21-002 | Expose overlay lag metrics and job completion hooks for Cartographer. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-21-201 | Implement graph build worker invoking Cartographer APIs. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-21-202 | Build overlay refresh worker batching policy/SBOM change events. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Observability Guilds | SCHED-WORKER-21-203 | Expose metrics/logs for graph scheduling and overlay lag. | -| Sprint 21 | Graph Explorer v1 | ops/devops/TASKS.md | TODO | DevOps & Cartographer Guilds | DEVOPS-GRAPH-21-001 | Add perf/load jobs for graph APIs and dashboards/alerts. | -| Sprint 21 | Graph Explorer v1 | ops/devops/TASKS.md | TODO | DevOps & UI Guilds | DEVOPS-GRAPH-21-002 | Integrate golden screenshots/JSON exports for graph visual regressions. | -| Sprint 21 | Graph Explorer v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-GRAPH-21-003 | Package Cartographer + SBOM Service into offline kit with seeded data. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & Cartographer Guilds | DOCS-GRAPH-21-001 | Publish `/docs/graph/overview.md` with reviewer checklist. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-GRAPH-21-002 | Write `/docs/graph/schema.md` covering node/edge/overlay schemas. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & BE-Base Guilds | DOCS-GRAPH-21-003 | Produce `/docs/graph/api.md` with endpoints, parameters, errors. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & UI Guilds | DOCS-GRAPH-21-004 | Create `/docs/ui/graph-explorer.md` detailing screens/flows. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & CLI Guilds | DOCS-GRAPH-21-005 | Document CLI commands in `/docs/cli/graph.md`. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & Architecture Guilds | DOCS-GRAPH-21-006 | Draft `/docs/architecture/cartographer.md` with sequence diagrams. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & Observability Guilds | DOCS-GRAPH-21-007 | Publish `/docs/observability/graph.md` metrics/traces/logs. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & Security Guilds | DOCS-GRAPH-21-008 | Write `/docs/security/graph-access.md` covering RBAC/tenancy. | -| Sprint 21 | Graph Explorer v1 | docs/TASKS.md | TODO | Docs & Cartographer Guilds | DOCS-GRAPH-21-009 | Populate `/docs/examples/graph/` with sample SBOMs, exports, screenshots. | -| Sprint 21 | Graph Explorer v1 | samples/TASKS.md | TODO | Samples & Cartographer Guilds | SAMPLES-GRAPH-21-001 | Produce sample graph fixtures (JSON/GraphML/layout tiles) for tests/docs. | -| Sprint 21 | Graph Explorer v1 | samples/TASKS.md | TODO | Samples & UI Guilds | SAMPLES-GRAPH-21-002 | Capture golden Graph Explorer screenshots and path exports. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench & Cartographer Guilds | BENCH-GRAPH-21-001 | Build graph viewport/path benchmark harness with baselines. | -| Sprint 21 | Graph Explorer v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench & UI Guilds | BENCH-GRAPH-21-002 | Add headless UI performance benchmark for Graph Explorer canvas. | +| Sprint 22 | Link-Not-Merge v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-LNM-22-001 | Publish advisories aggregation doc with observation/linkset philosophy. | +| Sprint 22 | Link-Not-Merge v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-LNM-22-002 | Publish VEX aggregation doc describing observation/linkset flow. | +| Sprint 22 | Link-Not-Merge v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-LNM-22-005 | Document UI evidence panel with conflict badges/AOC drill-down. | +| Sprint 22 | Link-Not-Merge v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LNM-22-001 | Execute advisory observation/linkset migration/backfill and automation. | +| Sprint 22 | Link-Not-Merge v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LNM-22-002 | Run VEX observation/linkset migration/backfill with monitoring/runbook. | +| Sprint 22 | Link-Not-Merge v1 | samples/TASKS.md | TODO | Samples Guild | SAMPLES-LNM-22-001 | Add advisory observation/linkset fixtures with conflicts. | +| Sprint 22 | Link-Not-Merge v1 | samples/TASKS.md | TODO | Samples Guild | SAMPLES-LNM-22-002 | Add VEX observation/linkset fixtures with status disagreements. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-AOC-22-001 | Roll out new advisory/vex ingest/read scopes. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Bench/TASKS.md | DONE (2025-10-26) | Bench Guild | BENCH-LNM-22-001 | Benchmark advisory observation ingest/correlation throughput. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Bench/TASKS.md | DONE (2025-10-26) | Bench Guild | BENCH-LNM-22-002 | Benchmark VEX ingest/correlation latency and event emission. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-LNM-22-001 | Implement advisory observation/linkset CLI commands with JSON/OSV export. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-LNM-22-002 | Implement VEX observation/linkset CLI commands. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-LNM-21-001 | Define immutable advisory observation schema with AOC metadata. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild, Data Science Guild | CONCELIER-LNM-21-002 | Implement advisory linkset builder with correlation signals/conflicts. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.Merge/TASKS.md | TODO | BE-Merge | MERGE-LNM-21-002 | Deprecate merge service and enforce observation-only pipeline. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage Guild | CONCELIER-LNM-21-101 | Provision observations/linksets collections and indexes. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Concelier Storage & DevOps Guilds | CONCELIER-LNM-21-102 | Backfill legacy merged advisories into observations/linksets with rollback tooling. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-LNM-21-201 | Ship advisory observation read APIs with pagination/RBAC. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-LNM-21-202 | Implement advisory linkset read/export/evidence endpoints mapped to `ERR_AGG_*`. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Concelier.Merge/TASKS.md | TODO | BE-Merge | MERGE-LNM-21-002 | Deprecate merge service and enforce observation-only pipeline. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-LNM-21-001 | Define immutable VEX observation model. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-LNM-21-002 | Build VEX linkset correlator with confidence/conflict recording. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Excititor Storage Guild | EXCITITOR-LNM-21-101 | Provision VEX observation/linkset collections and indexes. | @@ -249,58 +198,21 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-202 | Implement VEX linkset endpoints + exports with evidence payloads. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-40-001 | Update severity selection to handle multiple source severities per linkset. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Excititor Guild | POLICY-ENGINE-40-002 | Integrate VEX linkset conflicts into effective findings/explain traces. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-LNM-21-001 | Surface advisory observation/linkset APIs through gateway with RBAC. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-LNM-21-002 | Expose VEX observation/linkset endpoints with export handling. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-LNM-21-001 | Update report/runtime payloads to consume linksets and surface source evidence. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-LNM-22-001 | Deliver Evidence panel with policy banner and source observations. | | Sprint 22 | Link-Not-Merge v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-LNM-22-003 | Add VEX evidence tab with conflict indicators and exports. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-LNM-22-001 | Implement advisory observation/linkset CLI commands with JSON/OSV export. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-LNM-22-002 | Implement VEX observation/linkset CLI commands. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-AOC-19-001 | Roll out new advisory/vex ingest/read scopes. | -| Sprint 22 | Link-Not-Merge v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LNM-22-001 | Execute advisory observation/linkset migration/backfill and automation. | -| Sprint 22 | Link-Not-Merge v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LNM-22-002 | Run VEX observation/linkset migration/backfill with monitoring/runbook. | -| Sprint 22 | Link-Not-Merge v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-LNM-22-001 | Publish advisories aggregation doc with observation/linkset philosophy. | -| Sprint 22 | Link-Not-Merge v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-LNM-22-002 | Publish VEX aggregation doc describing observation/linkset flow. | -| Sprint 22 | Link-Not-Merge v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-LNM-22-005 | Document UI evidence panel with conflict badges/AOC drill-down. | -| Sprint 22 | Link-Not-Merge v1 | samples/TASKS.md | TODO | Samples Guild | SAMPLES-LNM-22-001 | Add advisory observation/linkset fixtures with conflicts. | -| Sprint 22 | Link-Not-Merge v1 | samples/TASKS.md | TODO | Samples Guild | SAMPLES-LNM-22-002 | Add VEX observation/linkset fixtures with status disagreements. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild | BENCH-LNM-22-001 | Benchmark advisory observation ingest/correlation throughput. | -| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild | BENCH-LNM-22-002 | Benchmark VEX ingest/correlation latency and event emission. | -| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONSOLE-23-001 | Ship `/console/dashboard` + `/console/filters` aggregates with tenant scoping and deterministic totals. | -| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, Scheduler Guild | WEB-CONSOLE-23-002 | Provide `/console/status` polling and `/console/runs/{id}/stream` SSE proxy with heartbeat/backoff. | -| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, Policy Guild | WEB-CONSOLE-23-003 | Expose `/console/exports` orchestration for evidence bundles, CSV/JSON streaming, manifest retrieval. | -| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONSOLE-23-004 | Implement `/console/search` fan-out router for CVE/GHSA/PURL/SBOM lookups with caching and RBAC. | -| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, DevOps Guild | WEB-CONSOLE-23-005 | Serve `/console/downloads` manifest with signed image metadata and offline guidance. | -| Sprint 23 | StellaOps Console | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-CONSOLE-23-001 | Register Console OIDC client with PKCE, scopes, short-lived tokens, and offline defaults. | -| Sprint 23 | StellaOps Console | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-CONSOLE-23-002 | Provide tenant catalog/user profile endpoints with audit logging and fresh-auth requirements. | -| Sprint 23 | StellaOps Console | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-CONSOLE-23-003 | Update security docs/sample configs for Console flows, CSP, and session policies. | -| Sprint 23 | StellaOps Console | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-CONSOLE-23-001 | Optimize findings/explain APIs for Console filters, aggregation hints, and provenance traces. | -| Sprint 23 | StellaOps Console | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Product Ops | POLICY-CONSOLE-23-002 | Expose simulation diff + approval state metadata for policy workspace scenarios. | -| Sprint 23 | StellaOps Console | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Scheduler Guild | EXPORT-CONSOLE-23-001 | Implement evidence bundle/export generator with signed manifests and telemetry. | -| Sprint 23 | StellaOps Console | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-CONSOLE-23-001 | Deliver Console SBOM catalog API with filters, evaluation metadata, and raw projections. | -| Sprint 23 | StellaOps Console | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-CONSOLE-23-002 | Provide component lookup/neighborhood endpoints for global search and overlays. | -| Sprint 23 | StellaOps Console | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-001 | Surface `/console/advisories` aggregation views with per-source metadata and filters. | -| Sprint 23 | StellaOps Console | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-002 | Provide advisory delta metrics API for dashboard + live status ticker. | -| Sprint 23 | StellaOps Console | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-003 | Add search helpers for CVE/GHSA/PURL lookups returning evidence fragments. | -| Sprint 23 | StellaOps Console | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Expose `/console/vex` aggregation endpoints with precedence and provenance. | -| Sprint 23 | StellaOps Console | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-002 | Publish VEX override delta metrics feeding dashboard/status ticker. | -| Sprint 23 | StellaOps Console | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-003 | Implement VEX search helpers for global search and explain drill-downs. | -| Sprint 23 | StellaOps Console | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-CONSOLE-23-001 | Extend runs API with SSE progress, queue lag summaries, RBAC actions, and history pagination. | -| Sprint 23 | StellaOps Console | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-CONSOLE-23-201 | Stream run progress events with heartbeat/dedupe for Console SSE consumers. | -| Sprint 23 | StellaOps Console | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-CONSOLE-23-202 | Coordinate evidence bundle job queueing, status tracking, cancellation, and retention. | -| Sprint 23 | StellaOps Console | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONSOLE-23-001 | Stand up console CI pipeline (pnpm cache, lint, tests, Playwright, Lighthouse, offline runners). | -| Sprint 23 | StellaOps Console | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONSOLE-23-002 | Deliver `stella-console` container + Helm overlays with SBOM/provenance and offline packaging. | -| Sprint 23 | StellaOps Console | ops/deployment/TASKS.md | TODO | Deployment Guild | DOWNLOADS-CONSOLE-23-001 | Maintain signed downloads manifest pipeline feeding Console + docs parity checks. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-001 | Publish `/docs/ui/console-overview.md` (IA, tenant model, filters, AOC alignment). | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-002 | Author `/docs/ui/navigation.md` with route map, filters, keyboard shortcuts, deep links. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-003 | Document `/docs/ui/sbom-explorer.md` covering catalog, graph, overlays, exports. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-004 | Produce `/docs/ui/advisories-and-vex.md` detailing aggregation-not-merge UX. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-005 | Write `/docs/ui/findings.md` with filters, explain, exports, CLI parity notes. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-006 | Publish `/docs/ui/policies.md` (editor, simulation, approvals, RBAC). | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-007 | Document `/docs/ui/runs.md` with SSE monitoring, diff, retries, evidence downloads. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-008 | Draft `/docs/ui/admin.md` covering tenants, roles, tokens, integrations, fresh-auth. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-009 | Publish `/docs/ui/downloads.md` aligning manifest with commands and offline flow. | -| Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-010 | Write `/docs/deploy/console.md` (Helm, ingress, TLS, env vars, health checks). | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-LNM-21-001 | Surface advisory observation/linkset APIs through gateway with RBAC. | +| Sprint 22 | Link-Not-Merge v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-LNM-21-002 | Expose VEX observation/linkset endpoints with export handling. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-001 | Publish `/docs/ui/console-overview.md` (IA, tenant model, filters, AOC alignment). | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-002 | Author `/docs/ui/navigation.md` with route map, filters, keyboard shortcuts, deep links. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-003 | Document `/docs/ui/sbom-explorer.md` covering catalog, graph, overlays, exports. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-004 | Produce `/docs/ui/advisories-and-vex.md` detailing aggregation-not-merge UX. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-005 | Write `/docs/ui/findings.md` with filters, explain, exports, CLI parity notes. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-006 | Publish `/docs/ui/policies.md` (editor, simulation, approvals, RBAC). | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-007 | Document `/docs/ui/runs.md` with SSE monitoring, diff, retries, evidence downloads. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-26) | Docs Guild | DOCS-CONSOLE-23-008 | Draft `/docs/ui/admin.md` covering tenants, roles, tokens, integrations, fresh-auth. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-27) | Docs Guild | DOCS-CONSOLE-23-009 | Publish `/docs/ui/downloads.md` aligning manifest with commands and offline flow. | +| Sprint 23 | StellaOps Console | docs/TASKS.md | DONE (2025-10-27) | Docs Guild, Deployment Guild, Console Guild | DOCS-CONSOLE-23-010 | Write `/docs/deploy/console.md` (Helm, ingress, TLS, env vars, health checks). | | Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-011 | Update `/docs/install/docker.md` to include console image, compose/Helm/offline examples. | | Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-012 | Publish `/docs/security/console-security.md` covering OIDC, scopes, CSP, evidence handling. | | Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-013 | Write `/docs/observability/ui-telemetry.md` cataloguing metrics/logs/dashboards/alerts. | @@ -308,40 +220,38 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-015 | Produce `/docs/architecture/console.md` describing packages, data flow, SSE design. | | Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-016 | Refresh `/docs/accessibility.md` with console keyboard flows, tokens, testing tools. | | Sprint 23 | StellaOps Console | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-23-017 | Create `/docs/examples/ui-tours.md` walkthroughs with annotated screenshots/GIFs. | +| Sprint 23 | StellaOps Console | ops/deployment/TASKS.md | TODO | Deployment Guild | DOWNLOADS-CONSOLE-23-001 | Maintain signed downloads manifest pipeline feeding Console + docs parity checks. | +| Sprint 23 | StellaOps Console | ops/devops/TASKS.md | BLOCKED (2025-10-26) | DevOps Guild | DEVOPS-CONSOLE-23-001 | Stand up console CI pipeline (pnpm cache, lint, tests, Playwright, Lighthouse, offline runners). | +| Sprint 23 | StellaOps Console | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONSOLE-23-002 | Deliver `stella-console` container + Helm overlays with SBOM/provenance and offline packaging. | +| Sprint 23 | StellaOps Console | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-CONSOLE-23-001 | Register Console OIDC client with PKCE, scopes, short-lived tokens, and offline defaults. | +| Sprint 23 | StellaOps Console | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-CONSOLE-23-002 | Provide tenant catalog/user profile endpoints with audit logging and fresh-auth requirements. | +| Sprint 23 | StellaOps Console | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-CONSOLE-23-003 | Update security docs/sample configs for Console flows, CSP, and session policies. | +| Sprint 23 | StellaOps Console | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-001 | Surface `/console/advisories` aggregation views with per-source metadata and filters. | +| Sprint 23 | StellaOps Console | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-002 | Provide advisory delta metrics API for dashboard + live status ticker. | +| Sprint 23 | StellaOps Console | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-CONSOLE-23-003 | Add search helpers for CVE/GHSA/PURL lookups returning evidence fragments. | +| Sprint 23 | StellaOps Console | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-001 | Expose `/console/vex` aggregation endpoints with precedence and provenance. | +| Sprint 23 | StellaOps Console | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-002 | Publish VEX override delta metrics feeding dashboard/status ticker. | +| Sprint 23 | StellaOps Console | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-CONSOLE-23-003 | Implement VEX search helpers for global search and explain drill-downs. | +| Sprint 23 | StellaOps Console | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Scheduler Guild | EXPORT-CONSOLE-23-001 | Implement evidence bundle/export generator with signed manifests and telemetry. | +| Sprint 23 | StellaOps Console | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-CONSOLE-23-001 | Optimize findings/explain APIs for Console filters, aggregation hints, and provenance traces. | +| Sprint 23 | StellaOps Console | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild, Product Ops | POLICY-CONSOLE-23-002 | Expose simulation diff + approval state metadata for policy workspace scenarios. | +| Sprint 23 | StellaOps Console | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-CONSOLE-23-001 | Deliver Console SBOM catalog API with filters, evaluation metadata, and raw projections. | +| Sprint 23 | StellaOps Console | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-CONSOLE-23-002 | Provide component lookup/neighborhood endpoints for global search and overlays. | +| Sprint 23 | StellaOps Console | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-CONSOLE-23-001 | Extend runs API with SSE progress, queue lag summaries, RBAC actions, and history pagination. | +| Sprint 23 | StellaOps Console | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-CONSOLE-23-201 | Stream run progress events with heartbeat/dedupe for Console SSE consumers. | +| Sprint 23 | StellaOps Console | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-CONSOLE-23-202 | Coordinate evidence bundle job queueing, status tracking, cancellation, and retention. | +| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONSOLE-23-001 | Ship `/console/dashboard` + `/console/filters` aggregates with tenant scoping and deterministic totals. | +| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, Scheduler Guild | WEB-CONSOLE-23-002 | Provide `/console/status` polling and `/console/runs/{id}/stream` SSE proxy with heartbeat/backoff. | +| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, Policy Guild | WEB-CONSOLE-23-003 | Expose `/console/exports` orchestration for evidence bundles, CSV/JSON streaming, manifest retrieval. | +| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONSOLE-23-004 | Implement `/console/search` fan-out router for CVE/GHSA/PURL/SBOM lookups with caching and RBAC. | +| Sprint 23 | StellaOps Console | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, DevOps Guild | WEB-CONSOLE-23-005 | Serve `/console/downloads` manifest with signed image metadata and offline guidance. | +| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-VULN-24-001 | Extend scopes (`vuln:read`) and signed permalinks. | +| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Concelier.Core/TASKS.md | DOING (2025-10-27) | Concelier Core Guild | CONCELIER-GRAPH-24-001 | Surface raw advisory observations/linksets for overlay services (no derived aggregation in ingestion). | +| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-GRAPH-24-001 | Surface raw VEX statements/linksets for overlay services (no suppression/precedence logic here). | | Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-60-001 | Maintain Redis effective decision maps for overlays. | | Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-60-002 | Provide simulation bridge for graph what-if APIs. | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-24-001 | Implement graph endpoints with pagination/ETags/RBAC. | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-24-002 | Gateway proxy for Policy/Vuln overlays (no simulation logic in gateway). | | Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-GRAPH-24-001 | Build Graph Explorer canvas with virtualization. | | Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-GRAPH-24-002 | Implement overlays (Policy/Evidence/License/Exposure). | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-24-001 | Add graph show/search/diff CLI commands. | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-GRAPH-24-001 | Extend scopes (`vuln:read`) and signed permalinks. | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-GRAPH-24-001 | Surface raw advisory observations/linksets for overlay services (no derived aggregation in ingestion). | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-GRAPH-24-001 | Surface raw VEX statements/linksets for overlay services (no suppression/precedence logic here). | -| Sprint 24 | Graph & Vuln Explorer v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-GRAPH-24-001 | Load test graph APIs and publish dashboards. | -| Sprint 24 | Graph & Vuln Explorer v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-GRAPH-24-001 | Document SBOM Graph Explorer UI. | -| Sprint 24 | Graph & Vuln Explorer v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-GRAPH-24-004 | Publish API docs for graph/vuln endpoints. | -| Sprint 24 | Graph & Vuln Explorer v1 | samples/TASKS.md | TODO | Samples Guild | SAMPLES-GRAPH-24-003 | Create large graph fixture for perf testing. | -| Sprint 24 | Graph & Vuln Explorer v1 | src/StellaOps.Bench/TASKS.md | TODO | Bench Guild | BENCH-GRAPH-24-001 | Build graph performance benchmark suite. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-EXC-25-001 | Extend SPL schema to reference exception effects and routing. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-001 | Add exception evaluation layer with specificity + effects. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-002 | Create exception collections/bindings storage + repos. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-003 | Implement Redis exception cache + invalidation. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-004 | Add metrics/tracing/logging for exception application. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-005 | Hook workers/events for activation/expiry. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXC-25-001 | Ship exception CRUD + workflow API endpoints. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXC-25-002 | Extend policy endpoints to include exception metadata. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXC-25-003 | Emit exception events/notifications with rate limits. | -| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-001 | Deliver Exception Center (list/kanban) with workflows. | -| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-002 | Build exception creation wizard with scope/timebox guardrails. | -| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-003 | Add inline exception drafting/proposing from explorers. | -| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-004 | Surface badges/countdowns/explain integration. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXC-25-001 | Implement CLI exception workflow commands. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXC-25-002 | Extend policy simulate with exception overrides. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-EXC-25-001 | Introduce exception scopes and routing matrix with MFA. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-EXC-25-002 | Update docs/config samples for exception governance. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-25-101 | Implement exception lifecycle worker for activation/expiry. | -| Sprint 25 | Exceptions v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-25-102 | Add expiring notification job & metrics. | | Sprint 25 | Exceptions v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXC-25-001 | Document exception governance concepts/workflow. | | Sprint 25 | Exceptions v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXC-25-002 | Document approvals routing / MFA requirements. | | Sprint 25 | Exceptions v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXC-25-003 | Publish API documentation for exceptions endpoints. | @@ -349,30 +259,25 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 25 | Exceptions v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXC-25-005 | Document UI exception center + badges. | | Sprint 25 | Exceptions v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXC-25-006 | Update CLI docs for exception commands. | | Sprint 25 | Exceptions v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXC-25-007 | Write migration guide for governed exceptions. | -| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-001 | Stand up Signals API skeleton with RBAC + health checks. | -| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-002 | Implement callgraph ingestion/normalization pipeline. | -| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-003 | Ingest runtime facts and persist context data with AOC provenance. | -| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-004 | Deliver reachability scoring engine writing reachability facts. | -| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-005 | Implement caches + signals events. | -| Sprint 26 | Reachability v1 | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-SPL-24-001 | Extend SPL schema with reachability predicates/actions. | -| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-001 | Integrate reachability inputs into policy evaluation and explainers. | -| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-002 | Optimize reachability fact retrieval + cache. | -| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-003 | Update SPL compiler for reachability predicates. | -| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-004 | Emit reachability metrics/traces. | -| Sprint 26 | Reachability v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-SIG-26-001 | Expose signals proxy endpoints with pagination and RBAC. | -| Sprint 26 | Reachability v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-SIG-26-002 | Join reachability data into policy/vuln responses. | -| Sprint 26 | Reachability v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-SIG-26-003 | Support reachability overrides in simulate APIs. | -| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-001 | Add reachability columns/badges to Vulnerability Explorer. | -| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-002 | Enhance Why drawer with call path/timeline. | -| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-003 | Add reachability overlay/time slider to SBOM Graph. | -| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-004 | Build Reachability Center + missing sensor view. | -| Sprint 26 | Reachability v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SIG-26-001 | Implement reachability CLI commands (upload/list/explain). | -| Sprint 26 | Reachability v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SIG-26-002 | Add reachability overrides to policy simulate. | -| Sprint 26 | Reachability v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-SIG-26-001 | Add signals scopes/roles + AOC requirements. | -| Sprint 26 | Reachability v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-26-201 | Implement reachability joiner worker. | -| Sprint 26 | Reachability v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-26-202 | Implement staleness monitor + notifications. | -| Sprint 26 | Reachability v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-SIG-26-001 | Provision pipelines/deployments for Signals service. | -| Sprint 26 | Reachability v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-SIG-26-002 | Add dashboards/alerts for reachability metrics. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-EXC-25-001 | Introduce exception scopes and routing matrix with MFA. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-EXC-25-002 | Update docs/config samples for exception governance. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXC-25-001 | Implement CLI exception workflow commands. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXC-25-002 | Extend policy simulate with exception overrides. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-001 | Add exception evaluation layer with specificity + effects. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-002 | Create exception collections/bindings storage + repos. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-003 | Implement Redis exception cache + invalidation. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-004 | Add metrics/tracing/logging for exception application. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-70-005 | Hook workers/events for activation/expiry. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-EXC-25-001 | Extend SPL schema to reference exception effects and routing. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-25-101 | Implement exception lifecycle worker for activation/expiry. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-25-102 | Add expiring notification job & metrics. | +| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-001 | Deliver Exception Center (list/kanban) with workflows. | +| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-002 | Build exception creation wizard with scope/timebox guardrails. | +| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-003 | Add inline exception drafting/proposing from explorers. | +| Sprint 25 | Exceptions v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-EXC-25-004 | Surface badges/countdowns/explain integration. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXC-25-001 | Ship exception CRUD + workflow API endpoints. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXC-25-002 | Extend policy endpoints to include exception metadata. | +| Sprint 25 | Exceptions v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXC-25-003 | Emit exception events/notifications with rate limits. | | Sprint 26 | Reachability v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-SIG-26-001 | Document reachability concepts and scoring. | | Sprint 26 | Reachability v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-SIG-26-002 | Document callgraph formats. | | Sprint 26 | Reachability v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-SIG-26-003 | Document runtime facts ingestion. | @@ -381,46 +286,32 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 26 | Reachability v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-SIG-26-006 | Document CLI reachability commands. | | Sprint 26 | Reachability v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-SIG-26-007 | Publish API docs for signals endpoints. | | Sprint 26 | Reachability v1 | docs/TASKS.md | TODO | Docs Guild | DOCS-SIG-26-008 | Write migration guide for enabling reachability. | +| Sprint 26 | Reachability v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-SIG-26-001 | Provision pipelines/deployments for Signals service. | +| Sprint 26 | Reachability v1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-SIG-26-002 | Add dashboards/alerts for reachability metrics. | +| Sprint 26 | Reachability v1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-SIG-26-001 | Add signals scopes/roles + AOC requirements. | +| Sprint 26 | Reachability v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SIG-26-001 | Implement reachability CLI commands (upload/list/explain). | +| Sprint 26 | Reachability v1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SIG-26-002 | Add reachability overrides to policy simulate. | | Sprint 26 | Reachability v1 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-SIG-26-001 | Expose advisory symbol metadata for signals scoring. | | Sprint 26 | Reachability v1 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-SIG-26-001 | Surface vendor exploitability hints to Signals. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-001 | Define Policy Registry OpenAPI spec for workspaces, versions, reviews, simulations, promotions, attestations. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-002 | Implement workspace storage + CRUD with tenant retention policies. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-003 | Integrate compile pipeline storing diagnostics, symbol tables, complexity metrics. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-004 | Deliver quick simulation API with limits and deterministic outputs. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & Scheduler Guilds | REGISTRY-API-27-005 | Build batch simulation orchestration, reduction, and evidence bundle storage. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-006 | Implement review workflow with comments, required approvers, webhooks. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & Security Guilds | REGISTRY-API-27-007 | Ship publish/sign pipeline with attestations, immutable versions. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-008 | Implement promotion/canary bindings per tenant/environment with rollback. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & Observability Guilds | REGISTRY-API-27-009 | Instrument metrics/logs/traces for compile, simulation, approval latency. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & QA Guilds | REGISTRY-API-27-010 | Build unit/integration/load test suites and seeded fixtures. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-001 | Return rule coverage, symbol table, docs, hashes from compile endpoint. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-002 | Enhance simulate outputs with heatmap, explain traces, delta summaries. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-003 | Enforce complexity/time limits with diagnostics. | -| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-004 | Update tests/fixtures for coverage, symbol table, explain, complexity. | -| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-27-001 | Proxy Policy Registry APIs with tenant scoping, RBAC, evidence streaming. | -| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-27-002 | Implement review lifecycle routes with audit logs and webhooks. | -| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Scheduler Guilds | WEB-POLICY-27-003 | Expose quick/batch simulation endpoints with SSE progress + manifests. | -| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Security Guilds | WEB-POLICY-27-004 | Add publish/promote/rollback endpoints with canary + signing enforcement. | -| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Observability Guilds | WEB-POLICY-27-005 | Instrument Policy Studio metrics/logs for dashboards. | -| Sprint 27 | Policy Studio | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-POLICY-27-001 | Define Policy Studio roles/scopes for author/review/approve/operate/audit. | -| Sprint 27 | Policy Studio | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guilds | AUTH-POLICY-27-002 | Wire signing service + fresh-auth enforcement for publish/promote. | -| Sprint 27 | Policy Studio | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-POLICY-27-003 | Update authority configuration/docs for Policy Studio roles & signing. | -| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-CONSOLE-27-001 | Provide policy simulation orchestration endpoints with SSE + RBAC. | -| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService & Observability Guilds | SCHED-CONSOLE-27-002 | Emit policy simulation telemetry endpoints/metrics + webhooks. | -| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-27-301 | Implement batch simulation worker sharding SBOMs with retries/backoff. | -| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-27-302 | Build reducer job aggregating shard outputs into manifests with checksums. | -| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Security Guilds | SCHED-WORKER-27-303 | Enforce tenant isolation/attestation integration and secret scanning for jobs. | -| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-001 | Implement policy workspace CLI commands (init, lint, compile, test). | -| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-002 | Add version bump, submit, review/approve CLI workflow commands. | -| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-003 | Extend simulate command for quick/batch runs, manifests, CI reports. | -| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-004 | Implement publish/promote/rollback/sign CLI lifecycle commands. | -| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI & Docs Guilds | CLI-POLICY-27-005 | Update CLI docs/reference for Policy Studio commands and schemas. | -| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-POLICY-27-001 | Add CI stage for policy lint/compile/test + secret scanning and artifacts. | -| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps & Policy Registry Guilds | DEVOPS-POLICY-27-002 | Provide optional batch simulation CI job with drift gating + PR comment. | -| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps & Security Guilds | DEVOPS-POLICY-27-003 | Manage signing keys + attestation verification in pipelines. | -| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps & Observability Guilds | DEVOPS-POLICY-27-004 | Build dashboards/alerts for compile latency, queue depth, approvals, promotions. | -| Sprint 27 | Policy Studio | ops/deployment/TASKS.md | TODO | Deployment & Policy Registry Guilds | DEPLOY-POLICY-27-001 | Create Helm/Compose overlays for Policy Registry + workers with signing config. | -| Sprint 27 | Policy Studio | ops/deployment/TASKS.md | TODO | Deployment & Policy Guilds | DEPLOY-POLICY-27-002 | Document policy rollout/rollback playbooks in runbook. | +| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-001 | Integrate reachability inputs into policy evaluation and explainers. | +| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-002 | Optimize reachability fact retrieval + cache. | +| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-003 | Update SPL compiler for reachability predicates. | +| Sprint 26 | Reachability v1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-80-004 | Emit reachability metrics/traces. | +| Sprint 26 | Reachability v1 | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-SPL-24-001 | Extend SPL schema with reachability predicates/actions. | +| Sprint 26 | Reachability v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-26-201 | Implement reachability joiner worker. | +| Sprint 26 | Reachability v1 | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-26-202 | Implement staleness monitor + notifications. | +| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-001 | Stand up Signals API skeleton with RBAC + health checks. | +| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-002 | Implement callgraph ingestion/normalization pipeline. | +| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-003 | Ingest runtime facts and persist context data with AOC provenance. | +| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-004 | Deliver reachability scoring engine writing reachability facts. | +| Sprint 26 | Reachability v1 | src/StellaOps.Signals/TASKS.md | TODO | Signals Guild | SIGNALS-24-005 | Implement caches + signals events. | +| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-001 | Add reachability columns/badges to Vulnerability Explorer. | +| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-002 | Enhance Why drawer with call path/timeline. | +| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-003 | Add reachability overlay/time slider to SBOM Graph. | +| Sprint 26 | Reachability v1 | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SIG-26-004 | Build Reachability Center + missing sensor view. | +| Sprint 26 | Reachability v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-SIG-26-001 | Expose signals proxy endpoints with pagination and RBAC. | +| Sprint 26 | Reachability v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-SIG-26-002 | Join reachability data into policy/vuln responses. | +| Sprint 26 | Reachability v1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-SIG-26-003 | Support reachability overrides in simulate APIs. | | Sprint 27 | Policy Studio | docs/TASKS.md | TODO | Docs & Policy Guilds | DOCS-POLICY-27-001 | Publish `/docs/policy/studio-overview.md` with lifecycle + roles. | | Sprint 27 | Policy Studio | docs/TASKS.md | TODO | Docs & Console Guilds | DOCS-POLICY-27-002 | Write `/docs/policy/authoring.md` with templates/snippets/lint rules. | | Sprint 27 | Policy Studio | docs/TASKS.md | TODO | Docs & Policy Registry Guilds | DOCS-POLICY-27-003 | Document `/docs/policy/versioning-and-publishing.md`. | @@ -435,47 +326,44 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 27 | Policy Studio | docs/TASKS.md | TODO | Docs & Ops Guilds | DOCS-POLICY-27-012 | Write `/docs/runbooks/policy-incident.md`. | | Sprint 27 | Policy Studio | docs/TASKS.md | TODO | Docs & Policy Guilds | DOCS-POLICY-27-013 | Update `/docs/examples/policy-templates.md` with new templates/snippets. | | Sprint 27 | Policy Studio | docs/TASKS.md | TODO | Docs & Policy Registry Guilds | DOCS-POLICY-27-014 | Refresh `/docs/aoc/aoc-guardrails.md` with Studio guardrails. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001 | Define node/edge schemas, identity rules, and fixtures for graph ingestion. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002 | Implement SBOM ingest consumer generating artifact/package/file nodes & edges. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-003 | Serve advisory overlay tiles from Conseiller linksets (no mutation of raw node/edge stores). | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-004 | Integrate VEX statements for `vex_exempts` edges with precedence metadata. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & Policy Guilds | GRAPH-INDEX-28-005 | Hydrate policy overlay nodes/edges referencing determinations + explains. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-006 | Produce graph snapshots per SBOM with lineage for diff jobs. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & Observability Guilds | GRAPH-INDEX-28-007 | Run clustering/centrality background jobs and persist cluster ids. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-008 | Build incremental/backfill pipeline with change streams, retries, backlog metrics. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & QA Guilds | GRAPH-INDEX-28-009 | Extend tests/perf fixtures ensuring determinism on large graphs. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & DevOps Guilds | GRAPH-INDEX-28-010 | Provide deployment/offline artifacts and docs for Graph Indexer. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-001 | Publish Graph API OpenAPI + JSON schemas for queries/tiles. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-002 | Implement `/graph/search` with caching and RBAC. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-003 | Build query planner + streaming tile pipeline with budgets. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-004 | Deliver `/graph/paths` with depth limits and policy overlay support. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-005 | Implement `/graph/diff` streaming adds/removes/changes for SBOM snapshots. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-006 | Compose advisory/VEX/policy overlays with caching + explain sampling. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-007 | Provide export jobs (GraphML/CSV/NDJSON/PNG/SVG) with manifests. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & Authority Guilds | GRAPH-API-28-008 | Enforce RBAC scopes, tenant headers, audit logging, rate limits. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & Observability Guilds | GRAPH-API-28-009 | Instrument metrics/logs/traces; publish dashboards. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & QA Guilds | GRAPH-API-28-010 | Build unit/integration/load tests with synthetic datasets. | -| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & DevOps Guilds | GRAPH-API-28-011 | Ship deployment/offline manifests + gateway integration docs. | -| Sprint 28 | Graph Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-24-001 | Route `/graph/*` APIs through gateway with tenant scoping and RBAC. | -| Sprint 28 | Graph Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-24-002 | Maintain overlay proxy routes to dedicated services (Policy/Vuln API), ensuring caching + RBAC only. | -| Sprint 28 | Graph Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Observability Guilds | WEB-GRAPH-24-004 | Add Graph Explorer telemetry endpoints and metrics aggregation. | -| Sprint 28 | Graph Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | WEB-LNM-21-001 | Provide advisory observation endpoints optimized for graph overlays. | -| Sprint 28 | Graph Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-GRAPH-24-101 | Deliver advisory summary API feeding graph tooltips. | -| Sprint 28 | Graph Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-101 | Provide VEX summary API for Graph Explorer inspector overlays. | -| Sprint 28 | Graph Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-30-001 | Finalize graph overlay contract + projection API. | -| Sprint 28 | Graph Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-30-002 | Implement simulation overlay bridge for Graph Explorer queries. | -| Sprint 28 | Graph Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy & Scheduler Guilds | POLICY-ENGINE-30-003 | Emit change events for effective findings supporting graph overlays. | -| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-21-001 | Provide graph build job APIs & overlay lag metrics. | -| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-21-201 | Run graph build worker for SBOM snapshots with retries/backoff. | -| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-21-202 | Execute overlay refresh worker subscribing to change events. | -| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Observability Guilds | SCHED-WORKER-21-203 | Emit metrics/logs for graph build/overlay jobs. | -| Sprint 28 | Graph Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-28-001 | Ship `stella sbom graph` subcommands (search, query, paths, diff, impacted, export) with JSON output + exit codes. | -| Sprint 28 | Graph Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-28-002 | Add saved query management + deep link helpers to CLI. | -| Sprint 28 | Graph Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-28-003 | Update CLI docs/examples for Graph Explorer commands. | -| Sprint 28 | Graph Explorer | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-GRAPH-28-001 | Configure load/perf tests, query budget alerts, and CI smoke for graph APIs. | -| Sprint 28 | Graph Explorer | ops/devops/TASKS.md | TODO | DevOps & Security Guilds | DEVOPS-GRAPH-28-002 | Implement caching/backpressure limits, rate limiting configs, and runaway query kill switches. | -| Sprint 28 | Graph Explorer | ops/devops/TASKS.md | TODO | DevOps & Observability Guilds | DEVOPS-GRAPH-28-003 | Build dashboards/alerts for tile latency, query denials, memory pressure. | -| Sprint 28 | Graph Explorer | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-GRAPH-28-001 | Provide deployment/offline instructions for Graph Indexer/API, including cache seeds. | +| Sprint 27 | Policy Studio | ops/deployment/TASKS.md | TODO | Deployment & Policy Registry Guilds | DEPLOY-POLICY-27-001 | Create Helm/Compose overlays for Policy Registry + workers with signing config. | +| Sprint 27 | Policy Studio | ops/deployment/TASKS.md | TODO | Deployment & Policy Guilds | DEPLOY-POLICY-27-002 | Document policy rollout/rollback playbooks in runbook. | +| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-POLICY-27-001 | Add CI stage for policy lint/compile/test + secret scanning and artifacts. | +| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps & Policy Registry Guilds | DEVOPS-POLICY-27-002 | Provide optional batch simulation CI job with drift gating + PR comment. | +| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps & Security Guilds | DEVOPS-POLICY-27-003 | Manage signing keys + attestation verification in pipelines. | +| Sprint 27 | Policy Studio | ops/devops/TASKS.md | TODO | DevOps & Observability Guilds | DEVOPS-POLICY-27-004 | Build dashboards/alerts for compile latency, queue depth, approvals, promotions. | +| Sprint 27 | Policy Studio | src/StellaOps.Authority/TASKS.md | TODO | Authority Core Guild | AUTH-POLICY-27-001 | Define Policy Studio roles/scopes for author/review/approve/operate/audit. | +| Sprint 27 | Policy Studio | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guilds | AUTH-POLICY-27-002 | Wire signing service + fresh-auth enforcement for publish/promote. | +| Sprint 27 | Policy Studio | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-POLICY-27-003 | Update authority configuration/docs for Policy Studio roles & signing. | +| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-001 | Implement policy workspace CLI commands (init, lint, compile, test). | +| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-002 | Add version bump, submit, review/approve CLI workflow commands. | +| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-003 | Extend simulate command for quick/batch runs, manifests, CI reports. | +| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-POLICY-27-004 | Implement publish/promote/rollback/sign CLI lifecycle commands. | +| Sprint 27 | Policy Studio | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI & Docs Guilds | CLI-POLICY-27-005 | Update CLI docs/reference for Policy Studio commands and schemas. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-001 | Return rule coverage, symbol table, docs, hashes from compile endpoint. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-002 | Enhance simulate outputs with heatmap, explain traces, delta summaries. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-003 | Enforce complexity/time limits with diagnostics. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-27-004 | Update tests/fixtures for coverage, symbol table, explain, complexity. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-001 | Define Policy Registry OpenAPI spec for workspaces, versions, reviews, simulations, promotions, attestations. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-002 | Implement workspace storage + CRUD with tenant retention policies. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-003 | Integrate compile pipeline storing diagnostics, symbol tables, complexity metrics. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-004 | Deliver quick simulation API with limits and deterministic outputs. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & Scheduler Guilds | REGISTRY-API-27-005 | Build batch simulation orchestration, reduction, and evidence bundle storage. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-006 | Implement review workflow with comments, required approvers, webhooks. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & Security Guilds | REGISTRY-API-27-007 | Ship publish/sign pipeline with attestations, immutable versions. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry Guild | REGISTRY-API-27-008 | Implement promotion/canary bindings per tenant/environment with rollback. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & Observability Guilds | REGISTRY-API-27-009 | Instrument metrics/logs/traces for compile, simulation, approval latency. | +| Sprint 27 | Policy Studio | src/StellaOps.Policy.Registry/TASKS.md | TODO | Policy Registry & QA Guilds | REGISTRY-API-27-010 | Build unit/integration/load test suites and seeded fixtures. | +| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-CONSOLE-27-001 | Provide policy simulation orchestration endpoints with SSE + RBAC. | +| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService & Observability Guilds | SCHED-CONSOLE-27-002 | Emit policy simulation telemetry endpoints/metrics + webhooks. | +| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-27-301 | Implement batch simulation worker sharding SBOMs with retries/backoff. | +| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-27-302 | Build reducer job aggregating shard outputs into manifests with checksums. | +| Sprint 27 | Policy Studio | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Security Guilds | SCHED-WORKER-27-303 | Enforce tenant isolation/attestation integration and secret scanning for jobs. | +| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-27-001 | Proxy Policy Registry APIs with tenant scoping, RBAC, evidence streaming. | +| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-POLICY-27-002 | Implement review lifecycle routes with audit logs and webhooks. | +| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Scheduler Guilds | WEB-POLICY-27-003 | Expose quick/batch simulation endpoints with SSE progress + manifests. | +| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Security Guilds | WEB-POLICY-27-004 | Add publish/promote/rollback endpoints with canary + signing enforcement. | +| Sprint 27 | Policy Studio | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Observability Guilds | WEB-POLICY-27-005 | Instrument Policy Studio metrics/logs for dashboards. | | Sprint 28 | Graph Explorer | docs/TASKS.md | TODO | Docs & SBOM Guilds | DOCS-GRAPH-28-001 | Publish `/docs/sbom/graph-explorer-overview.md`. | | Sprint 28 | Graph Explorer | docs/TASKS.md | TODO | Docs & Console Guilds | DOCS-GRAPH-28-002 | Write `/docs/sbom/graph-using-the-console.md` with walkthrough + accessibility tips. | | Sprint 28 | Graph Explorer | docs/TASKS.md | TODO | Docs & Graph API Guilds | DOCS-GRAPH-28-003 | Document `/docs/sbom/graph-query-language.md` (JSON schema, cost rules). | @@ -488,61 +376,51 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 28 | Graph Explorer | docs/TASKS.md | TODO | Docs & Observability Guilds | DOCS-GRAPH-28-010 | Publish `/docs/observability/graph-telemetry.md`. | | Sprint 28 | Graph Explorer | docs/TASKS.md | TODO | Docs & Ops Guilds | DOCS-GRAPH-28-011 | Write `/docs/runbooks/graph-incidents.md`. | | Sprint 28 | Graph Explorer | docs/TASKS.md | TODO | Docs & Security Guilds | DOCS-GRAPH-28-012 | Create `/docs/security/graph-rbac.md`. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-29-001 | Design ledger & projection schemas, hashing strategy, and migrations for Findings Ledger. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-29-002 | Implement ledger write API with hash chaining and Merkle root anchoring job. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Scheduler Guilds | LEDGER-29-003 | Build projector worker deriving `findings_projection` with idempotent replay. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Policy Guilds | LEDGER-29-004 | Integrate Policy Engine batch evaluation into projector with rationale caching. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-29-005 | Implement workflow mutation endpoints producing ledger events (assign/comment/accept-risk/etc.). | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Security Guilds | LEDGER-29-006 | Add attachment encryption, signed URLs, and CSRF protections for workflow endpoints. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Observability Guilds | LEDGER-29-007 | Instrument ledger metrics/logs/alerts (write latency, projection lag, anchoring). | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & QA Guilds | LEDGER-29-008 | Provide replay/determinism/load tests for ledger/projector pipelines. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & DevOps Guilds | LEDGER-29-009 | Deliver deployment/offline artefacts, backup/restore, Merkle anchoring guidance. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-001 | Publish Vuln Explorer OpenAPI + query schemas. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-002 | Implement list/query endpoints with grouping, paging, cost budgets. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-003 | Implement detail endpoint combining evidence, policy rationale, paths, history. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Findings Ledger Guilds | VULN-API-29-004 | Expose workflow APIs writing ledger events with validation + idempotency. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Policy Guilds | VULN-API-29-005 | Implement policy simulation endpoint producing diffs without side effects. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-006 | Integrate Graph Explorer paths metadata and deep-link parameters. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Security Guilds | VULN-API-29-007 | Enforce RBAC/ABAC, CSRF, attachment security, and audit logging. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-008 | Provide evidence bundle export job with signing + manifests. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Observability Guilds | VULN-API-29-009 | Instrument API telemetry (latency, workflow counts, exports). | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & QA Guilds | VULN-API-29-010 | Deliver unit/integration/perf/determinism tests for Vuln Explorer API. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & DevOps Guilds | VULN-API-29-011 | Ship deployment/offline manifests, health checks, scaling docs. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001 | Normalize advisory keys, persist `links[]`, backfill, and expose raw payload snapshots. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-VULN-29-002 | Provide advisory evidence retrieval endpoint for Vuln Explorer. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService & Observability Guilds | CONCELIER-VULN-29-004 | Add metrics/logs/events for advisory normalization supporting resolver. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Canonicalize VEX keys and product scopes with backfill + links. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-002 | Expose VEX evidence retrieval endpoint for Explorer evidence tabs. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService & Observability Guilds | EXCITITOR-VULN-29-004 | Instrument metrics/logs for VEX normalization and suppression events. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-VULN-29-001 | Emit inventory evidence with scope/runtime/path/safe version hints; publish change events. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service & Findings Ledger Guilds | SBOM-VULN-29-002 | Provide resolver feed for candidate generation with idempotent delivery. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-001 | Implement policy batch evaluation endpoint returning determinations + rationale. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-002 | Provide simulation diff API for Vuln Explorer comparisons. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-003 | Include path/scope annotations in determinations for Explorer. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild & Observability Guild | POLICY-ENGINE-29-004 | Add telemetry for batch evaluation + simulation jobs. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-001 | Route `/vuln/*` APIs with tenant RBAC, ABAC, anti-forgery enforcement. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-002 | Proxy workflow calls to Findings Ledger with correlation IDs + retries. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-003 | Expose simulation/export orchestration with SSE/progress + signed links. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Observability Guilds | WEB-VULN-29-004 | Aggregate Vuln Explorer telemetry (latency, errors, exports). | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-VULN-29-001 | Define Vuln Explorer RBAC/ABAC scopes and issuer metadata. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-VULN-29-002 | Enforce CSRF, attachment signing, and audit logging referencing ledger hashes. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-VULN-29-003 | Update docs/config samples for Vuln Explorer roles and security posture. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-VULN-29-001 | Expose resolver job APIs + status monitoring for Vuln Explorer recomputation. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService & Observability Guilds | SCHED-VULN-29-002 | Provide projector lag metrics endpoint + webhook notifications. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-29-001 | Implement resolver worker applying ecosystem version semantics and path scope. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-29-002 | Implement evaluation worker invoking Policy Engine and updating ledger queues. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Observability Guilds | SCHED-WORKER-29-003 | Add monitoring for resolver/evaluation backlog and SLA alerts. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-001 | Implement `stella vuln list` with grouping, filters, JSON/CSV output. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-002 | Implement `stella vuln show` with evidence/policy/path display. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-003 | Add workflow CLI commands (assign/comment/accept-risk/verify-fix/target-fix/reopen). | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-004 | Implement `stella vuln simulate` producing diff summaries/Markdown. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-005 | Implement `stella vuln export` and bundle signature verification. | -| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI & Docs Guilds | CLI-VULN-29-006 | Update CLI docs/examples for Vulnerability Explorer commands. | -| Sprint 29 | Vulnerability Explorer | ops/devops/TASKS.md | TODO | DevOps & Findings Ledger Guilds | DEVOPS-VULN-29-001 | Set up CI/backups/anchoring monitoring for Findings Ledger. | -| Sprint 29 | Vulnerability Explorer | ops/devops/TASKS.md | TODO | DevOps & Vuln Explorer API Guilds | DEVOPS-VULN-29-002 | Configure Vuln Explorer perf tests, budgets, dashboards, alerts. | -| Sprint 29 | Vulnerability Explorer | ops/devops/TASKS.md | TODO | DevOps & Console Guilds | DEVOPS-VULN-29-003 | Integrate Vuln Explorer telemetry pipeline with privacy safeguards + dashboards. | -| Sprint 29 | Vulnerability Explorer | ops/deployment/TASKS.md | TODO | Deployment & Findings Ledger Guilds | DEPLOY-VULN-29-001 | Provide deployments for Findings Ledger/projector with migrations/backups. | -| Sprint 29 | Vulnerability Explorer | ops/deployment/TASKS.md | TODO | Deployment & Vuln Explorer API Guilds | DEPLOY-VULN-29-002 | Package Vuln Explorer API deployments/health checks/offline kit notes. | +| Sprint 28 | Graph Explorer | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-GRAPH-28-001 | Provide deployment/offline instructions for Graph Indexer/API, including cache seeds. | +| Sprint 28 | Graph Explorer | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-GRAPH-28-001 | Configure load/perf tests, query budget alerts, and CI smoke for graph APIs. | +| Sprint 28 | Graph Explorer | ops/devops/TASKS.md | TODO | DevOps & Security Guilds | DEVOPS-GRAPH-28-002 | Implement caching/backpressure limits, rate limiting configs, and runaway query kill switches. | +| Sprint 28 | Graph Explorer | ops/devops/TASKS.md | TODO | DevOps & Observability Guilds | DEVOPS-GRAPH-28-003 | Build dashboards/alerts for tile latency, query denials, memory pressure. | +| Sprint 28 | Graph Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-28-001 | Ship `stella sbom graph` subcommands (search, query, paths, diff, impacted, export) with JSON output + exit codes. | +| Sprint 28 | Graph Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-28-002 | Add saved query management + deep link helpers to CLI. | +| Sprint 28 | Graph Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-GRAPH-28-003 | Update CLI docs/examples for Graph Explorer commands. | +| Sprint 28 | Graph Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-GRAPH-24-101 | Deliver advisory summary API feeding graph tooltips. | +| Sprint 28 | Graph Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-GRAPH-28-102 | Add batch fetch for advisory observations/linksets keyed by component sets to feed Graph overlay tooltips efficiently. | +| Sprint 28 | Graph Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | WEB-LNM-21-001 | Provide advisory observation endpoints optimized for graph overlays. | +| Sprint 28 | Graph Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-101 | Provide VEX summary API for Graph Explorer inspector overlays. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-001 | Publish Graph API OpenAPI + JSON schemas for queries/tiles. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-002 | Implement `/graph/search` with caching and RBAC. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-003 | Build query planner + streaming tile pipeline with budgets. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-004 | Deliver `/graph/paths` with depth limits and policy overlay support. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-005 | Implement `/graph/diff` streaming adds/removes/changes for SBOM snapshots. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-006 | Compose advisory/VEX/policy overlays with caching + explain sampling. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API Guild | GRAPH-API-28-007 | Provide export jobs (GraphML/CSV/NDJSON/PNG/SVG) with manifests. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & Authority Guilds | GRAPH-API-28-008 | Enforce RBAC scopes, tenant headers, audit logging, rate limits. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & Observability Guilds | GRAPH-API-28-009 | Instrument metrics/logs/traces; publish dashboards. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & QA Guilds | GRAPH-API-28-010 | Build unit/integration/load tests with synthetic datasets. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Api/TASKS.md | TODO | Graph API & DevOps Guilds | GRAPH-API-28-011 | Ship deployment/offline manifests + gateway integration docs. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001 | Define node/edge schemas, identity rules, and fixtures for graph ingestion. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002 | Implement SBOM ingest consumer generating artifact/package/file nodes & edges. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-003 | Serve advisory overlay tiles from Conseiller linksets (no mutation of raw node/edge stores). | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-004 | Integrate VEX statements for `vex_exempts` edges with precedence metadata. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & Policy Guilds | GRAPH-INDEX-28-005 | Hydrate policy overlay nodes/edges referencing determinations + explains. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-006 | Produce graph snapshots per SBOM with lineage for diff jobs. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & Observability Guilds | GRAPH-INDEX-28-007 | Run clustering/centrality background jobs and persist cluster ids. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer Guild | GRAPH-INDEX-28-008 | Build incremental/backfill pipeline with change streams, retries, backlog metrics. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & QA Guilds | GRAPH-INDEX-28-009 | Extend tests/perf fixtures ensuring determinism on large graphs. | +| Sprint 28 | Graph Explorer | src/StellaOps.Graph.Indexer/TASKS.md | TODO | Graph Indexer & DevOps Guilds | GRAPH-INDEX-28-010 | Provide deployment/offline artifacts and docs for Graph Indexer. | +| Sprint 28 | Graph Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-30-001 | Finalize graph overlay contract + projection API. | +| Sprint 28 | Graph Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-30-002 | Implement simulation overlay bridge for Graph Explorer queries. | +| Sprint 28 | Graph Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy & Scheduler Guilds | POLICY-ENGINE-30-003 | Emit change events for effective findings supporting graph overlays. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-21-001 | Provide graph build/overlay job APIs; see `docs/SCHED-WEB-21-001-GRAPH-APIS.md`. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-21-002 | Provide overlay lag metrics endpoint/webhook; see `docs/SCHED-WEB-21-001-GRAPH-APIS.md`. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | DONE (2025-10-26) | Scheduler WebService Guild, Authority Core Guild | SCHED-WEB-21-003 | Replace header auth with Authority scopes using `StellaOpsScopes`; dev fallback only when `Scheduler:Authority:Enabled=false`. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | DOING (2025-10-26) | Scheduler WebService Guild, Scheduler Storage Guild | SCHED-WEB-21-004 | Persist graph jobs + emit completion events/webhook. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-21-201 | Run graph build worker for SBOM snapshots with retries/backoff. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-21-202 | Execute overlay refresh worker subscribing to change events. | +| Sprint 28 | Graph Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Observability Guilds | SCHED-WORKER-21-203 | Emit metrics/logs for graph build/overlay jobs. | +| Sprint 28 | Graph Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-24-001 | Route `/graph/*` APIs through gateway with tenant scoping and RBAC. | +| Sprint 28 | Graph Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-GRAPH-24-002 | Maintain overlay proxy routes to dedicated services (Policy/Vuln API), ensuring caching + RBAC only. | +| Sprint 28 | Graph Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Observability Guilds | WEB-GRAPH-24-004 | Add Graph Explorer telemetry endpoints and metrics aggregation. | | Sprint 29 | Vulnerability Explorer | docs/TASKS.md | TODO | Docs Guild | DOCS-VULN-29-001 | Publish `/docs/vuln/explorer-overview.md`. | | Sprint 29 | Vulnerability Explorer | docs/TASKS.md | TODO | Docs & Console Guilds | DOCS-VULN-29-002 | Write `/docs/vuln/explorer-using-console.md`. | | Sprint 29 | Vulnerability Explorer | docs/TASKS.md | TODO | Docs Guild | DOCS-VULN-29-003 | Author `/docs/vuln/explorer-api.md`. | @@ -556,6 +434,81 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 29 | Vulnerability Explorer | docs/TASKS.md | TODO | Docs & Security Guilds | DOCS-VULN-29-011 | Publish `/docs/security/vuln-rbac.md`. | | Sprint 29 | Vulnerability Explorer | docs/TASKS.md | TODO | Docs & Ops Guilds | DOCS-VULN-29-012 | Publish `/docs/runbooks/vuln-ops.md`. | | Sprint 29 | Vulnerability Explorer | docs/TASKS.md | TODO | Docs & Deployment Guilds | DOCS-VULN-29-013 | Update `/docs/install/containers.md` with Findings Ledger & Vuln Explorer API. | +| Sprint 29 | Vulnerability Explorer | ops/deployment/TASKS.md | TODO | Deployment & Findings Ledger Guilds | DEPLOY-VULN-29-001 | Provide deployments for Findings Ledger/projector with migrations/backups. | +| Sprint 29 | Vulnerability Explorer | ops/deployment/TASKS.md | TODO | Deployment & Vuln Explorer API Guilds | DEPLOY-VULN-29-002 | Package Vuln Explorer API deployments/health checks/offline kit notes. | +| Sprint 29 | Vulnerability Explorer | ops/devops/TASKS.md | TODO | DevOps & Findings Ledger Guilds | DEVOPS-VULN-29-001 | Set up CI/backups/anchoring monitoring for Findings Ledger. | +| Sprint 29 | Vulnerability Explorer | ops/devops/TASKS.md | TODO | DevOps & Vuln Explorer API Guilds | DEVOPS-VULN-29-002 | Configure Vuln Explorer perf tests, budgets, dashboards, alerts. | +| Sprint 29 | Vulnerability Explorer | ops/devops/TASKS.md | TODO | DevOps & Console Guilds | DEVOPS-VULN-29-003 | Integrate Vuln Explorer telemetry pipeline with privacy safeguards + dashboards. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-VULN-29-001 | Define Vuln Explorer RBAC/ABAC scopes and issuer metadata. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-VULN-29-002 | Enforce CSRF, attachment signing, and audit logging referencing ledger hashes. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Docs Guild | AUTH-VULN-29-003 | Update docs/config samples for Vuln Explorer roles and security posture. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-001 | Implement `stella vuln list` with grouping, filters, JSON/CSV output. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-002 | Implement `stella vuln show` with evidence/policy/path display. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-003 | Add workflow CLI commands (assign/comment/accept-risk/verify-fix/target-fix/reopen). | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-004 | Implement `stella vuln simulate` producing diff summaries/Markdown. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-005 | Implement `stella vuln export` and bundle signature verification. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI & Docs Guilds | CLI-VULN-29-006 | Update CLI docs/examples for Vulnerability Explorer commands. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001 | Canonicalize (lossless) advisory identifiers, persist `links[]`, backfill, and expose raw payload snapshots (no merge/derived fields). | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-VULN-29-002 | Provide advisory evidence retrieval endpoint for Vuln Explorer. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService & Observability Guilds | CONCELIER-VULN-29-004 | Add metrics/logs/events for advisory normalization supporting resolver. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Canonicalize (lossless) VEX keys and product scopes with backfill + links (no merge/suppression). | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-002 | Expose VEX evidence retrieval endpoint for Explorer evidence tabs. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService & Observability Guilds | EXCITITOR-VULN-29-004 | Instrument metrics/logs for VEX normalization and suppression events. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-29-001 | Design ledger & projection schemas, hashing strategy, and migrations for Findings Ledger. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-29-002 | Implement ledger write API with hash chaining and Merkle root anchoring job. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Scheduler Guilds | LEDGER-29-003 | Build projector worker deriving `findings_projection` with idempotent replay. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Policy Guilds | LEDGER-29-004 | Integrate Policy Engine batch evaluation into projector with rationale caching. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-29-005 | Implement workflow mutation endpoints producing ledger events (assign/comment/accept-risk/etc.). | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Security Guilds | LEDGER-29-006 | Add attachment encryption, signed URLs, and CSRF protections for workflow endpoints. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & Observability Guilds | LEDGER-29-007 | Instrument ledger metrics/logs/alerts (write latency, projection lag, anchoring). | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & QA Guilds | LEDGER-29-008 | Provide replay/determinism/load tests for ledger/projector pipelines. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger & DevOps Guilds | LEDGER-29-009 | Deliver deployment/offline artefacts, backup/restore, Merkle anchoring guidance. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-001 | Implement policy batch evaluation endpoint returning determinations + rationale. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-002 | Provide simulation diff API for Vuln Explorer comparisons. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-003 | Include path/scope annotations in determinations for Explorer. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild & Observability Guild | POLICY-ENGINE-29-004 | Add telemetry for batch evaluation + simulation jobs. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-VULN-29-001 | Emit inventory evidence with scope/runtime/path/safe version hints; publish change events. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service & Findings Ledger Guilds | SBOM-VULN-29-002 | Provide resolver feed for candidate generation with idempotent delivery. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-VULN-29-001 | Expose resolver job APIs + status monitoring for Vuln Explorer recomputation. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService & Observability Guilds | SCHED-VULN-29-002 | Provide projector lag metrics endpoint + webhook notifications. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-29-001 | Implement resolver worker applying ecosystem version semantics and path scope. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-29-002 | Implement evaluation worker invoking Policy Engine and updating ledger queues. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker & Observability Guilds | SCHED-WORKER-29-003 | Add monitoring for resolver/evaluation backlog and SLA alerts. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-001 | Publish Vuln Explorer OpenAPI + query schemas. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-002 | Implement list/query endpoints with grouping, paging, cost budgets. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-003 | Implement detail endpoint combining evidence, policy rationale, paths, history. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Findings Ledger Guilds | VULN-API-29-004 | Expose workflow APIs writing ledger events with validation + idempotency. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Policy Guilds | VULN-API-29-005 | Implement policy simulation endpoint producing diffs without side effects. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-006 | Integrate Graph Explorer paths metadata and deep-link parameters. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Security Guilds | VULN-API-29-007 | Enforce RBAC/ABAC, CSRF, attachment security, and audit logging. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API Guild | VULN-API-29-008 | Provide evidence bundle export job with signing + manifests. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & Observability Guilds | VULN-API-29-009 | Instrument API telemetry (latency, workflow counts, exports). | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & QA Guilds | VULN-API-29-010 | Deliver unit/integration/perf/determinism tests for Vuln Explorer API. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.VulnExplorer.Api/TASKS.md | TODO | Vuln Explorer API & DevOps Guilds | VULN-API-29-011 | Ship deployment/offline manifests, health checks, scaling docs. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-001 | Route `/vuln/*` APIs with tenant RBAC, ABAC, anti-forgery enforcement. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-002 | Proxy workflow calls to Findings Ledger with correlation IDs + retries. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-003 | Expose simulation/export orchestration with SSE/progress + signed links. | +| Sprint 29 | Vulnerability Explorer | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform & Observability Guilds | WEB-VULN-29-004 | Aggregate Vuln Explorer telemetry (latency, errors, exports). | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-001 | Publish `/docs/vex/consensus-overview.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-002 | Write `/docs/vex/consensus-algorithm.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-003 | Document `/docs/vex/issuer-directory.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-004 | Publish `/docs/vex/consensus-api.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-005 | Create `/docs/vex/consensus-console.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-006 | Add `/docs/policy/vex-trust-model.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-007 | Author `/docs/sbom/vex-mapping.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-008 | Publish `/docs/security/vex-signatures.md`. | +| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-009 | Write `/docs/runbooks/vex-ops.md`. | +| Sprint 30 | VEX Lens | ops/devops/TASKS.md | TODO | DevOps Guild | VEXLENS-30-009, ISSUER-30-005 | Set up CI/perf/telemetry dashboards for VEX Lens and Issuer Directory. | +| Sprint 30 | VEX Lens | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | VEXLENS-30-007 | Implement `stella vex consensus` CLI commands with list/show/simulate/export. | +| Sprint 30 | VEX Lens | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild, VEX Lens Guild | CONCELIER-VEXLENS-30-001 | Guarantee advisory key consistency and provide cross-links for consensus rationale (VEX Lens). | +| Sprint 30 | VEX Lens | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Ensure VEX evidence includes issuer hints, signatures, product trees for Lens consumption. | +| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory Guild | ISSUER-30-001 | Implement issuer CRUD API with RBAC and audit logs. | +| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & Security Guilds | ISSUER-30-002 | Implement key management endpoints with expiry enforcement. | +| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & Policy Guilds | ISSUER-30-003 | Provide trust weight override APIs with audit trails. | +| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & VEX Lens Guilds | ISSUER-30-004 | Integrate issuer data into signature verification clients. | +| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & Observability Guilds | ISSUER-30-005 | Instrument issuer change metrics/logs and dashboards. | +| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & DevOps Guilds | ISSUER-30-006 | Provide deployment/backup/offline docs for Issuer Directory. | +| Sprint 30 | VEX Lens | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-30-101 | Surface trust weighting configuration (issuer weights, modifiers, decay) for VEX Lens via Policy Studio/API. | | Sprint 30 | VEX Lens | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-30-001 | Implement VEX normalization pipeline (CSAF, OpenVEX, CycloneDX) with deterministic outputs. | | Sprint 30 | VEX Lens | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-30-002 | Build product mapping library aligning CSAF product trees to purls/versions with scope scoring. | | Sprint 30 | VEX Lens | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens & Issuer Directory Guilds | VEXLENS-30-003 | Integrate signature verification using issuer keys; annotate evidence. | @@ -567,27 +520,12 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 30 | VEX Lens | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens & Observability Guilds | VEXLENS-30-009 | Instrument metrics/logs/traces; publish dashboards/alerts. | | Sprint 30 | VEX Lens | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens & QA Guilds | VEXLENS-30-010 | Build unit/property/integration/load tests and determinism harness. | | Sprint 30 | VEX Lens | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens & DevOps Guilds | VEXLENS-30-011 | Provide deployment manifests, scaling guides, offline seeds, runbooks. | -| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory Guild | ISSUER-30-001 | Implement issuer CRUD API with RBAC and audit logs. | -| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & Security Guilds | ISSUER-30-002 | Implement key management endpoints with expiry enforcement. | -| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & Policy Guilds | ISSUER-30-003 | Provide trust weight override APIs with audit trails. | -| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & VEX Lens Guilds | ISSUER-30-004 | Integrate issuer data into signature verification clients. | -| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & Observability Guilds | ISSUER-30-005 | Instrument issuer change metrics/logs and dashboards. | -| Sprint 30 | VEX Lens | src/StellaOps.IssuerDirectory/TASKS.md | TODO | Issuer Directory & DevOps Guilds | ISSUER-30-006 | Provide deployment/backup/offline docs for Issuer Directory. | -| Sprint 30 | VEX Lens | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001 | Ensure VEX evidence includes issuer hints, signatures, product trees for Lens consumption. | -| Sprint 30 | VEX Lens | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001 | Guarantee advisory key consistency and provide cross-links for consensus rationale. | -| Sprint 30 | VEX Lens | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-29-001 | Extend policy config with trust weights/thresholds for consensus inputs. | -| Sprint 30 | VEX Lens | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-VULN-29-001, VEXLENS-30-007 | Route `/vex/consensus` APIs through gateway with RBAC/ABAC and telemetry. | -| Sprint 30 | VEX Lens | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | VEXLENS-30-007 | Implement `stella vex consensus` CLI commands with list/show/simulate/export. | -| Sprint 30 | VEX Lens | ops/devops/TASKS.md | TODO | DevOps Guild | VEXLENS-30-009, ISSUER-30-005 | Set up CI/perf/telemetry dashboards for VEX Lens and Issuer Directory. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-001 | Publish `/docs/vex/consensus-overview.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-002 | Write `/docs/vex/consensus-algorithm.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-003 | Document `/docs/vex/issuer-directory.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-004 | Publish `/docs/vex/consensus-api.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-005 | Create `/docs/vex/consensus-console.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-006 | Add `/docs/policy/vex-trust-model.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-007 | Author `/docs/sbom/vex-mapping.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-008 | Publish `/docs/security/vex-signatures.md`. | -| Sprint 30 | VEX Lens | docs/TASKS.md | TODO | Docs Guild | DOCS-VEX-30-009 | Write `/docs/runbooks/vex-ops.md`. | +| Sprint 30 | VEX Lens | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild, VEX Lens Guild | WEB-VEX-30-007 | Route `/vex/consensus` APIs via gateway with RBAC/ABAC, caching, and telemetry (proxy-only). | +| Sprint 31 | Advisory AI | docs/TASKS.md | TODO | Docs Guild | DOCS-AIAI-31-001 | Publish Advisory AI overview doc. | +| Sprint 31 | Advisory AI | docs/TASKS.md | TODO | Docs Guild | DOCS-AIAI-31-002 | Publish architecture doc for Advisory AI. | +| Sprint 31 | Advisory AI | docs/TASKS.md | TODO | Docs Guild | DOCS-AIAI-31-003..009 | Complete API/Console/CLI/Policy/Security/SBOM/Runbook docs. | +| Sprint 31 | Advisory AI | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-AIAI-31-001 | Provide Advisory AI deployment/offline guidance. | +| Sprint 31 | Advisory AI | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIAI-31-001 | Provision CI/perf/telemetry for Advisory AI. | | Sprint 31 | Advisory AI | src/StellaOps.AdvisoryAI/TASKS.md | TODO | Advisory AI Guild | AIAI-31-001 | Implement advisory/VEX retrievers with paragraph anchors and citations. | | Sprint 31 | Advisory AI | src/StellaOps.AdvisoryAI/TASKS.md | TODO | Advisory AI Guild | AIAI-31-002 | Build SBOM context retriever and blast radius estimator. | | Sprint 31 | Advisory AI | src/StellaOps.AdvisoryAI/TASKS.md | TODO | Advisory AI Guild | AIAI-31-003 | Deliver deterministic toolset (version checks, dependency analysis, policy lookup). | @@ -597,138 +535,141 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 31 | Advisory AI | src/StellaOps.AdvisoryAI/TASKS.md | TODO | Advisory AI & Observability Guilds | AIAI-31-007 | Instrument metrics/logs/traces and dashboards. | | Sprint 31 | Advisory AI | src/StellaOps.AdvisoryAI/TASKS.md | TODO | Advisory AI & DevOps Guilds | AIAI-31-008 | Package inference + deployment manifests/flags. | | Sprint 31 | Advisory AI | src/StellaOps.AdvisoryAI/TASKS.md | TODO | Advisory AI & QA Guilds | AIAI-31-009 | Build golden/injection/perf tests ensuring determinism. | -| Sprint 31 | Advisory AI | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-AIAI-31-001 | Expose advisory chunk API with paragraph anchors. | -| Sprint 31 | Advisory AI | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001 | Provide VEX chunks with justifications and signatures. | -| Sprint 31 | Advisory AI | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-AIAI-31-001 | Expose enriched rationale API for conflict explanations. | -| Sprint 31 | Advisory AI | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-AIAI-31-001 | Deliver SBOM path/timeline endpoints for Advisory AI. | -| Sprint 31 | Advisory AI | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-31-001 | Provide policy knobs for Advisory AI. | -| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-001 | Route `/advisory/ai/*` APIs with RBAC/telemetry. | -| Sprint 31 | Advisory AI | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-VULN-29-001, CLI-VEX-30-001 | Implement `stella advise *` CLI commands. | -| Sprint 31 | Advisory AI | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIAI-31-001 | Provision CI/perf/telemetry for Advisory AI. | -| Sprint 31 | Advisory AI | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-AIAI-31-001 | Provide Advisory AI deployment/offline guidance. | -| Sprint 31 | Advisory AI | docs/TASKS.md | TODO | Docs Guild | DOCS-AIAI-31-001 | Publish Advisory AI overview doc. | -| Sprint 31 | Advisory AI | docs/TASKS.md | TODO | Docs Guild | DOCS-AIAI-31-002 | Publish architecture doc for Advisory AI. | -| Sprint 31 | Advisory AI | docs/TASKS.md | TODO | Docs Guild | DOCS-AIAI-31-003..009 | Complete API/Console/CLI/Policy/Security/SBOM/Runbook docs. | -| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-001 | Route Advisory AI API endpoints with RBAC/ABAC and telemetry. | -| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-002 | Provide batch orchestration and retry handling for Advisory AI. | -| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-003 | Emit Advisory AI gateway telemetry/audit logs. | -| Sprint 31 | Advisory AI | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-31-001 | Expose Advisory AI policy parameters. | | Sprint 31 | Advisory AI | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-AIAI-31-001 | Define Advisory AI scopes and remote inference toggles. | | Sprint 31 | Advisory AI | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-AIAI-31-002 | Enforce prompt logging and consent/audit flows. | -| Sprint 31 | Advisory AI | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIAI-31-001 | Provision CI/perf/monitoring for Advisory AI. | -| Sprint 31 | Advisory AI | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-AIAI-31-001 | Ship Advisory AI deployment/offline kit guidance. | -| Sprint 31 | Advisory AI | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-AIAI-31-001 | Expose consensus rationale API enhancements for Advisory AI. | +| Sprint 31 | Advisory AI | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIAI-31-001 | Implement `stella advise *` CLI commands leveraging Advisory AI orchestration and policy scopes. | +| Sprint 31 | Advisory AI | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-AIAI-31-001 | Expose advisory chunk API with paragraph anchors. | +| Sprint 31 | Advisory AI | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001 | Provide VEX chunks with justifications and signatures. | +| Sprint 31 | Advisory AI | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-31-001 | Provide policy knobs for Advisory AI. | +| Sprint 31 | Advisory AI | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-AIAI-31-001 | Deliver SBOM path/timeline endpoints for Advisory AI. | +| Sprint 31 | Advisory AI | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-AIAI-31-001 | Expose enriched rationale API for conflict explanations. | | Sprint 31 | Advisory AI | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-AIAI-31-002 | Provide batching/caching hooks for Advisory AI. | -| Sprint 31 | Advisory AI | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-AIAI-31-001 | Supply VEX chunks with justifications and signatures. | -| Sprint 31 | Advisory AI | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-AIAI-31-001 | Expose advisory chunk API with anchors. | -| Sprint 31 | Advisory AI | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-AIAI-31-001 | Provide SBOM path/timeline endpoints for Advisory AI. | +| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-001 | Route `/advisory/ai/*` APIs with RBAC/telemetry. | +| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-002 | Provide batch orchestration and retry handling for Advisory AI. | +| Sprint 31 | Advisory AI | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-AIAI-31-003 | Emit Advisory AI gateway telemetry/audit logs. | +| Sprint 32 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-32-001 | Author `/docs/orchestrator/overview.md` covering mission, roles, AOC alignment, and imposed rule reminder. | +| Sprint 32 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-32-002 | Author `/docs/orchestrator/architecture.md` detailing scheduler, DAGs, rate limits, and data model. | +| Sprint 32 | Orchestrator Dashboard | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ORCH-32-001 | Provision staging Postgres/message-bus charts, CI smoke deploy, and baseline dashboards for queue depth and inflight jobs. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-ORCH-32-001 | Introduce `orch:read` scope and `Orch.Viewer` role with metadata, discovery docs, and offline defaults. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-32-001 | Register Concelier sources with orchestrator, publish schedules/rate policies, and seed metadata. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-32-002 | Embed worker SDK into Concelier ingestion loops emitting progress, heartbeats, and artifact hashes. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-ORCH-32-001 | Adopt worker SDK in Excititor worker with job claim/heartbeat and artifact summary emission. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-32-001 | Bootstrap Go worker SDK (client config, job claim, acknowledgement flow) with integration tests. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-32-002 | Add heartbeat/progress helpers, structured logging, and default metrics exporters to Go SDK. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-32-001 | Bootstrap Python async SDK with job claim/config adapters and sample worker. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-32-002 | Implement heartbeat/progress helpers and logging/metrics instrumentation for Python workers. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-32-001 | Bootstrap orchestrator service with Postgres schema/migrations for sources, runs, jobs, dag_edges, artifacts, quotas, schedules. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-32-002 | Implement scheduler DAG planner, dependency resolver, and job state machine for read-only tracking. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-32-003 | Expose read-only REST APIs (sources, runs, jobs, DAG) with OpenAPI + validation. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-32-004 | Ship WebSocket/SSE live update stream and metrics counters/histograms for job lifecycle. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-32-005 | Deliver worker claim/heartbeat/progress endpoints capturing artifact metadata and checksums. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-32-001 | Bootstrap Go worker SDK (client config, job claim, acknowledgement flow) with integration tests. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-32-002 | Add heartbeat/progress helpers, structured logging, and default metrics exporters to Go SDK. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-32-001 | Bootstrap Python async SDK with job claim/config adapters and sample worker. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-32-002 | Implement heartbeat/progress helpers and logging/metrics instrumentation for Python workers. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-32-001 | Register Concelier sources with orchestrator, publish schedules/rate policies, and seed metadata. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-32-002 | Embed worker SDK into Concelier ingestion loops emitting progress, heartbeats, and artifact hashes. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-ORCH-32-001 | Adopt worker SDK in Excititor worker with job claim/heartbeat and artifact summary emission. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-ORCH-32-001 | Integrate orchestrator job IDs into SBOM ingest/index pipelines with artifact hashing and status updates. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-32-101 | Define orchestrator `policy_eval` job contract, idempotency keys, and enqueue hooks for change events. | +| Sprint 32 | Orchestrator Dashboard | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-ORCH-32-001 | Integrate orchestrator job IDs into SBOM ingest/index pipelines with artifact hashing and status updates. | | Sprint 32 | Orchestrator Dashboard | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-ORCH-32-001 | Expose read-only orchestrator APIs via gateway with tenant scoping, caching headers, and rate limits. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-ORCH-32-001 | Introduce `orch:read` scope and `Orch.Viewer` role with metadata, discovery docs, and offline defaults. | -| Sprint 32 | Orchestrator Dashboard | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-ORCH-32-001 | Implement `stella orch sources|runs|jobs` list/show commands with filters, table/JSON output, and exit codes. | -| Sprint 32 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-32-001 | Author `/docs/orchestrator/overview.md` covering mission, roles, AOC alignment, and imposed rule reminder. | -| Sprint 32 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-32-002 | Author `/docs/orchestrator/architecture.md` detailing scheduler, DAGs, rate limits, and data model. | -| Sprint 32 | Orchestrator Dashboard | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ORCH-32-001 | Provision staging Postgres/message-bus charts, CI smoke deploy, and baseline dashboards for queue depth and inflight jobs. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-001 | Enable source/job control actions (test, pause/resume, retry/cancel/prioritize) with RBAC and audit hooks. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-002 | Implement adaptive token-bucket rate limiter and concurrency caps reacting to upstream 429/503 signals. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-003 | Add watermark/backfill manager with event-time windows, duplicate suppression, and preview API. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-004 | Deliver dead-letter storage, replay endpoints, and surfaced error classes with remediation hints. | +| Sprint 33 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-33-001 | Author `/docs/orchestrator/api.md` with endpoints, WebSocket events, error codes, and imposed rule reminder. | +| Sprint 33 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-33-002 | Author `/docs/orchestrator/console.md` covering screens, accessibility, and live updates. | +| Sprint 33 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-33-003 | Author `/docs/orchestrator/cli.md` with command reference, examples, and exit codes. | +| Sprint 33 | Governance & Rules | ops/devops/TASKS.md | DOING (2025-10-26) | DevOps Guild, Platform Leads | DEVOPS-RULES-33-001 | Contracts & Rules anchor (gateway proxy-only; Policy Engine overlays/simulations; AOC ingestion canonicalization; Graph Indexer + Graph API as sole platform). | +| Sprint 33 | Orchestrator Dashboard | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ORCH-33-001 | Publish Grafana dashboards for rate-limit/backpressure/error clustering and configure alert rules with runbooks. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-ORCH-33-001 | Add `Orch.Operator` role, control action scopes, and enforce reason/ticket field capture. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-33-001 | Wire orchestrator control hooks (pause, throttle, retry) into Concelier workers with safe checkpoints. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-ORCH-33-001 | Honor orchestrator throttles, classify VEX errors, and emit retry-safe checkpoints in Excititor worker. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-33-001 | Add artifact upload helpers (object store + checksum) and idempotency guard to Go SDK. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-33-002 | Implement error classification/retry helper and structured failure report in Go SDK. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-33-001 | Add artifact publish/idempotency features to Python SDK with object store integration. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-33-002 | Expose error classification/retry/backoff helpers in Python SDK with structured logging. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-33-001 | Wire orchestrator control hooks (pause, throttle, retry) into Concelier workers with safe checkpoints. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-ORCH-33-001 | Honor orchestrator throttles, classify VEX errors, and emit retry-safe checkpoints in Excititor worker. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-ORCH-33-001 | Report SBOM ingest backpressure metrics and support orchestrator pause/resume/backfill signals. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-001 | Enable source/job control actions (test, pause/resume, retry/cancel/prioritize) with RBAC and audit hooks. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-002 | Implement adaptive token-bucket rate limiter and concurrency caps reacting to upstream 429/503 signals. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-003 | Add watermark/backfill manager with event-time windows, duplicate suppression, and preview API. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-33-004 | Deliver dead-letter storage, replay endpoints, and surfaced error classes with remediation hints. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-33-101 | Implement orchestrator-driven policy evaluation workers with heartbeats, SLO metrics, and rate limit awareness. | +| Sprint 33 | Orchestrator Dashboard | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-ORCH-33-001 | Report SBOM ingest backpressure metrics and support orchestrator pause/resume/backfill signals. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-ORCH-33-001 | Expose `consensus_compute` orchestrator job type and integrate VEX Lens worker for diff batches. | | Sprint 33 | Orchestrator Dashboard | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-ORCH-33-001 | Add control endpoints (actions/backfill) and SSE bridging with permission checks and error mapping. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-ORCH-33-001 | Add `Orch.Operator` role, control action scopes, and enforce reason/ticket field capture. | -| Sprint 33 | Orchestrator Dashboard | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-ORCH-33-001 | Implement CLI action verbs (`sources pause|resume|test`, `jobs retry|cancel`, `jobs tail`) with streaming output. | -| Sprint 33 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-33-001 | Author `/docs/orchestrator/api.md` with endpoints, WebSocket events, error codes, and imposed rule reminder. | -| Sprint 33 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-33-002 | Author `/docs/orchestrator/console.md` covering screens, accessibility, and live updates. | -| Sprint 33 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-33-003 | Author `/docs/orchestrator/cli.md` with command reference, examples, and exit codes. | -| Sprint 33 | Orchestrator Dashboard | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ORCH-33-001 | Publish Grafana dashboards for rate-limit/backpressure/error clustering and configure alert rules with runbooks. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-001 | Implement quota management APIs, SLO burn-rate computation, and alert budget tracking. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-002 | Build audit log and immutable run ledger export with signed manifest support. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-003 | Run perf/scale validation (10k jobs, dispatch <150 ms) and add autoscaling hooks. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-004 | Package orchestrator container, Helm overlays, offline bundle seeds, and provenance attestations. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-34-001 | Add backfill range execution, watermark handshake, and artifact dedupe verification to Go SDK. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-34-001 | Add backfill support and deterministic artifact dedupe validation to Python SDK. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-34-001 | Implement orchestrator-driven backfills for advisory sources with idempotent artifact reuse and ledger linkage. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-ORCH-34-001 | Support orchestrator backfills and circuit breaker resets for Excititor sources with auditing. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-ORCH-34-001 | Enable SBOM backfill and watermark reconciliation; emit coverage metrics and flood guard. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-34-101 | Expose policy eval run ledger exports and SLO burn metrics to orchestrator. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-ORCH-34-001 | Integrate consensus compute completion events with orchestrator ledger and provenance outputs. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-34-101 | Link orchestrator run ledger entries into Findings Ledger provenance export and audit queries. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-ORCH-34-001 | Expose quotas/backfill/queue metrics endpoints, throttle toggles, and error clustering APIs. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-ORCH-34-001 | Add `Orch.Admin` role for quotas/backfills, enforce audit reason requirements, update docs and offline defaults. | -| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-ORCH-34-001 | Implement backfill wizard and quota management commands with dry-run preview and guardrails. | | Sprint 34 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-34-001 | Author `/docs/orchestrator/run-ledger.md` describing provenance export format and audits. | | Sprint 34 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-34-002 | Author `/docs/security/secrets-handling.md` covering KMS refs, redaction, and operator hygiene. | | Sprint 34 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-34-003 | Author `/docs/operations/orchestrator-runbook.md` (failures, backfill guide, circuit breakers). | | Sprint 34 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-34-004 | Author `/docs/schemas/artifacts.md` detailing artifact kinds, schema versions, hashing, storage layout. | | Sprint 34 | Orchestrator Dashboard | docs/TASKS.md | TODO | Docs Guild | DOCS-ORCH-34-005 | Author `/docs/slo/orchestrator-slo.md` defining SLOs, burn alerts, and measurement strategy. | -| Sprint 34 | Orchestrator Dashboard | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ORCH-34-001 | Harden production dashboards/alerts, synthetic probes, and incident response playbooks for orchestrator. | | Sprint 34 | Orchestrator Dashboard | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-ORCH-34-001 | Provide Helm/Compose manifests, scaling defaults, and offline kit instructions for orchestrator service. | +| Sprint 34 | Orchestrator Dashboard | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ORCH-34-001 | Harden production dashboards/alerts, synthetic probes, and incident response playbooks for orchestrator. | | Sprint 34 | Orchestrator Dashboard | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-34-006 | Bundle orchestrator service, worker SDK samples, and Postgres snapshot into Offline Kit with integrity checks. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-ORCH-34-001 | Add `Orch.Admin` role for quotas/backfills, enforce audit reason requirements, update docs and offline defaults. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-ORCH-34-001 | Implement backfill wizard and quota management commands with dry-run preview and guardrails. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-ORCH-34-001 | Implement orchestrator-driven backfills for advisory sources with idempotent artifact reuse and ledger linkage. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Excititor Worker Guild | EXCITITOR-ORCH-34-001 | Support orchestrator backfills and circuit breaker resets for Excititor sources with auditing. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-34-101 | Link orchestrator run ledger entries into Findings Ledger provenance export and audit queries. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Go/TASKS.md | TODO | Worker SDK Guild | WORKER-GO-34-001 | Add backfill range execution, watermark handshake, and artifact dedupe verification to Go SDK. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator.WorkerSdk.Python/TASKS.md | TODO | Worker SDK Guild | WORKER-PY-34-001 | Add backfill support and deterministic artifact dedupe validation to Python SDK. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-001 | Implement quota management APIs, SLO burn-rate computation, and alert budget tracking. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-002 | Build audit log and immutable run ledger export with signed manifest support. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-003 | Run perf/scale validation (10k jobs, dispatch <150 ms) and add autoscaling hooks. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-34-004 | Package orchestrator container, Helm overlays, offline bundle seeds, and provenance attestations. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-34-101 | Expose policy eval run ledger exports and SLO burn metrics to orchestrator. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.SbomService/TASKS.md | TODO | SBOM Service Guild | SBOM-ORCH-34-001 | Enable SBOM backfill and watermark reconciliation; emit coverage metrics and flood guard. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-ORCH-34-001 | Integrate consensus compute completion events with orchestrator ledger and provenance outputs. | +| Sprint 34 | Orchestrator Dashboard | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-ORCH-34-001 | Expose quotas/backfill/queue metrics endpoints, throttle toggles, and error clustering APIs. | +| Sprint 35 | EPDR Foundations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild | SCANNER-ANALYZERS-LANG-11-001 | Build entrypoint resolver (identity + environment profiles) and emit normalized entrypoint records. | +| Sprint 35 | EPDR Foundations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild | SCANNER-ANALYZERS-LANG-11-002 | Static IL/reflection/ALC heuristics producing dependency edges with reason codes and confidence. | +| Sprint 35 | EPDR Foundations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild, Signals Guild | SCANNER-ANALYZERS-LANG-11-003 | Runtime loader/PInvoke signal ingestion merged with static/declared edges (confidence & explain). | +| Sprint 35 | Export Center Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-35-001 | Author `/docs/export-center/overview.md` with purpose, profiles, security, and imposed rule reminder. | +| Sprint 35 | Export Center Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-35-002 | Author `/docs/export-center/architecture.md` detailing service components, adapters, manifests, signing, and distribution. | +| Sprint 35 | Export Center Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-35-003 | Publish `/docs/export-center/profiles.md` covering schemas, examples, and compatibility. | +| Sprint 35 | Export Center Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-EXPORT-35-001 | Package exporter service/worker containers, Helm overlays (download-only), and rollout guide. | +| Sprint 35 | Export Center Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-EXPORT-35-001 | Create exporter CI pipeline (lint/test/perf smoke), object storage fixtures, and initial Grafana dashboards. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-001 | Bootstrap exporter service, configuration, and migrations for export profiles/runs/inputs/distributions with tenant scopes. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-002 | Implement planner resolving filters to iterators and orchestrator job contract with deterministic sampling. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-003 | Deliver JSON adapters (raw/policy) with canonical normalization, redaction enforcement, and zstd writers. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-004 | Build mirror (full) adapter producing filesystem layout, manifests, and bundle assembly for download profile. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-005 | Implement manifest/provenance writer and KMS signing/attestation for export bundles. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-006 | Expose Export API (profiles, runs, download) with SSE updates, concurrency controls, and audit logging. | -| Sprint 35 | Export Center Phase 1 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-35-101 | Register export job type, quotas, and rate policies; surface export job telemetry for scheduler. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-EXPORT-35-001 | Provide paginated streaming endpoints for advisories, VEX, SBOMs, and findings filtered by scope selectors. | +| Sprint 35 | Export Center Phase 1 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-35-101 | Register export job type, quotas, and rate policies; surface export job telemetry for scheduler. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-35-201 | Expose deterministic policy snapshot + evaluated findings endpoint aligned with Export Center requirements. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.VexLens/TASKS.md | TODO | VEX Lens Guild | VEXLENS-EXPORT-35-001 | Publish consensus snapshot API delivering deterministic JSON for export consumption. | | Sprint 35 | Export Center Phase 1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXPORT-35-001 | Route Export Center APIs through gateway with tenant scoping, viewer/operator scopes, and streaming downloads. | -| Sprint 35 | Export Center Phase 1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-EXPORT-35-001 | Define `Export.Viewer|Operator|Admin` scopes/roles, issuer templates, and offline defaults. | -| Sprint 35 | Export Center Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXPORT-35-001 | Implement `stella export profiles|runs` list/show/create (download) with manifest retrieval and resume-aware downloads. | -| Sprint 35 | Export Center Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-35-001 | Author `/docs/export-center/overview.md` with purpose, profiles, security, and imposed rule reminder. | -| Sprint 35 | Export Center Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-35-002 | Author `/docs/export-center/architecture.md` detailing service components, adapters, manifests, signing, and distribution. | -| Sprint 35 | Export Center Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-35-003 | Publish `/docs/export-center/profiles.md` covering schemas, examples, and compatibility. | -| Sprint 35 | Export Center Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-EXPORT-35-001 | Create exporter CI pipeline (lint/test/perf smoke), object storage fixtures, and initial Grafana dashboards. | -| Sprint 35 | Export Center Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-EXPORT-35-001 | Package exporter service/worker containers, Helm overlays (download-only), and rollout guide. | +| Sprint 36 | EPDR Observations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild, SBOM Service Guild | SCANNER-ANALYZERS-LANG-11-004 | Normalize EPDR output to Scanner observation writer (entrypoints + edges + env profiles). | +| Sprint 36 | EPDR Observations | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | Scanner EPDR Guild, QA Guild | SCANNER-ANALYZERS-LANG-11-005 | End-to-end fixtures/benchmarks covering publish modes, RIDs, trimming, NativeAOT with explain traces. | +| Sprint 36 | Export Center Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-36-004 | Author `/docs/export-center/api.md` with endpoint examples and imposed rule note. | +| Sprint 36 | Export Center Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-36-005 | Publish `/docs/export-center/cli.md` covering commands, scripts, verification, and imposed rule reminder. | +| Sprint 36 | Export Center Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-36-006 | Write `/docs/export-center/trivy-adapter.md` detailing mappings, compatibility, and test matrix. | +| Sprint 36 | Export Center Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-EXPORT-36-001 | Document registry credentials, OCI push workflows, and automation for export distributions. | +| Sprint 36 | Export Center Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-EXPORT-36-001 | Integrate Trivy compatibility validation, OCI push smoke tests, and metrics dashboards for export throughput. | +| Sprint 36 | Export Center Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXPORT-36-001 | Add `stella export distribute` (OCI/objstore), `run download --resume`, and status polling enhancements. | | Sprint 36 | Export Center Phase 2 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-36-001 | Implement Trivy DB adapter (core) with schema mapping, validation, and compatibility gating. | | Sprint 36 | Export Center Phase 2 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-36-002 | Add Trivy Java DB variant, shared manifest entries, and adapter regression tests. | | Sprint 36 | Export Center Phase 2 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-36-003 | Build OCI distribution engine for exports with descriptor annotations and registry auth handling. | | Sprint 36 | Export Center Phase 2 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-36-004 | Extend planner/run lifecycle for OCI/object storage distributions with retry + idempotency. | | Sprint 36 | Export Center Phase 2 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-36-101 | Add distribution job follow-ups, retention metadata, and metrics for export runs. | | Sprint 36 | Export Center Phase 2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXPORT-36-001 | Expose distribution endpoints (OCI/object storage) and manifest/provenance download proxies with RBAC. | -| Sprint 36 | Export Center Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXPORT-36-001 | Add `stella export distribute` (OCI/objstore), `run download --resume`, and status polling enhancements. | -| Sprint 36 | Export Center Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-36-004 | Author `/docs/export-center/api.md` with endpoint examples and imposed rule note. | -| Sprint 36 | Export Center Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-36-005 | Publish `/docs/export-center/cli.md` covering commands, scripts, verification, and imposed rule reminder. | -| Sprint 36 | Export Center Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-36-006 | Write `/docs/export-center/trivy-adapter.md` detailing mappings, compatibility, and test matrix. | -| Sprint 36 | Export Center Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-EXPORT-36-001 | Integrate Trivy compatibility validation, OCI push smoke tests, and metrics dashboards for export throughput. | -| Sprint 36 | Export Center Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-EXPORT-36-001 | Document registry credentials, OCI push workflows, and automation for export distributions. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-001 | Implement mirror delta adapter, base export linkage, and content-addressed reuse. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-002 | Add bundle encryption, key wrapping with KMS, and verification tooling for encrypted exports. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-003 | Deliver scheduling/retention engine (cron/event triggers), audit trails, and retry idempotency enhancements. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-004 | Provide export verification API and CLI integration, including hash/signature validation endpoints. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-37-101 | Enable scheduled export runs, retention pruning hooks, and failure alerting integration. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXPORT-37-001 | Surface scheduling, retention, and verification endpoints plus encryption parameter handling. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-EXPORT-37-001 | Add `Export.Admin` scope enforcement for retention, encryption keys, and scheduling APIs. | -| Sprint 37 | Export Center Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXPORT-37-001 | Implement `stella export schedule`, `run verify`, and bundle verification tooling with signature/hash checks. | | Sprint 37 | Export Center Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-37-001 | Publish `/docs/export-center/mirror-bundles.md` detailing layouts, deltas, encryption, imposed rule reminder. | | Sprint 37 | Export Center Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-37-002 | Publish `/docs/export-center/provenance-and-signing.md` covering manifests, attestation, verification. | | Sprint 37 | Export Center Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-37-003 | Publish `/docs/operations/export-runbook.md` for failures, tuning, capacity, with imposed rule note. | | Sprint 37 | Export Center Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-EXPORT-37-004 | Publish `/docs/security/export-hardening.md` covering RBAC, isolation, encryption, and imposed rule. | | Sprint 37 | Export Center Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-EXPORT-37-001 | Finalize dashboards/alerts for exports (failure, verify), retention jobs, and chaos testing harness. | | Sprint 37 | Export Center Phase 3 | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-37-001 | Package Export Center mirror bundles + verification tooling into Offline Kit with manifest/signature updates. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-EXPORT-37-001 | Add `Export.Admin` scope enforcement for retention, encryption keys, and scheduling APIs. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-EXPORT-37-001 | Implement `stella export schedule`, `run verify`, and bundle verification tooling with signature/hash checks. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-001 | Implement mirror delta adapter, base export linkage, and content-addressed reuse. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-002 | Add bundle encryption, key wrapping with KMS, and verification tooling for encrypted exports. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-003 | Deliver scheduling/retention engine (cron/event triggers), audit trails, and retry idempotency enhancements. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-37-004 | Provide export verification API and CLI integration, including hash/signature validation endpoints. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-37-101 | Enable scheduled export runs, retention pruning hooks, and failure alerting integration. | +| Sprint 37 | Export Center Phase 3 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-EXPORT-37-001 | Surface scheduling, retention, and verification endpoints plus encryption parameter handling. | +| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-001 | Format detector & binary identity for ELF/PE/Mach-O (multi-slice) with stable entrypoint IDs. | +| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-002 | ELF dynamic parser emitting dtneeded edges, runpath metadata, symbol version needs. | +| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-003 | PE import + delay-load + SxS manifest parsing producing reason-coded edges. | +| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-004 | Mach-O load command parsing with @rpath expansion and slice handling. | +| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-005 | Cross-platform resolver engine modeling search order/explain traces for ELF/PE/Mach-O. | +| Sprint 37 | Native Analyzer Core | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-006 | Heuristic scanner for dlopen/LoadLibrary strings, plugin configs, ecosystem hints with confidence tags. | +| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild | SCANNER-ANALYZERS-NATIVE-20-007 | Serialize entrypoints/edges/env profiles to Scanner writer (AOC-compliant observations). | +| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild, QA Guild | SCANNER-ANALYZERS-NATIVE-20-008 | Fixture suite + determinism benchmarks for native analyzer across linux/windows/macos. | +| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-NATIVE-20-009 | Optional runtime capture adapters (eBPF/ETW/dyld) producing runtime-load edges with redaction. | +| Sprint 38 | Native Observation Pipeline | src/StellaOps.Scanner.Analyzers.Native/TASKS.md | TODO | Native Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-NATIVE-20-010 | Package native analyzer plug-in + Offline Kit updates and restart-time loading. | +| Sprint 38 | Notifications Studio Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-NOTIFY-38-001 | Publish `/docs/notifications/overview.md` and `/docs/notifications/architecture.md` ending with imposed rule statement. | +| Sprint 38 | Notifications Studio Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-NOTIFY-38-001 | Package notifier API/worker Helm overlays (email/chat/webhook), secrets templates, rollout guide. | +| Sprint 38 | Notifications Studio Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-38-001 | Stand up notifier CI pipelines, event bus fixtures, base dashboards for events/notifications latency. | +| Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-NOTIFY-38-001 | Implement `stella notify` rule/template/incident commands (list/create/test/ack) with file-based inputs. | | Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-38-001 | Bootstrap notifier service, migrations for notif tables, event ingestion, and rule engine foundation (policy violations + job failures). | | Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-38-002 | Implement channel adapters (email, chat-webhook, generic webhook) with retry and audit logging. | | Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-38-003 | Deliver template service (versioning, preview), rendering pipeline with redaction, and provenance links. | @@ -736,302 +677,302 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-38-101 | Standardize event envelope publication (policy/export/job lifecycle) with idempotency keys for notifier ingestion. | | Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-38-201 | Emit enriched violation events including rationale IDs via orchestrator bus. | | Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-NOTIFY-38-001 | Route notifier APIs through gateway with tenant scoping and operator scopes. | -| Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-NOTIFY-38-001 | Define `Notify.Viewer|Operator|Admin` scopes, issuer templates, and offline defaults. | -| Sprint 38 | Notifications Studio Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-NOTIFY-38-001 | Implement `stella notify` rule/template/incident commands (list/create/test/ack) with file-based inputs. | -| Sprint 38 | Notifications Studio Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-NOTIFY-38-001 | Publish `/docs/notifications/overview.md` and `/docs/notifications/architecture.md` ending with imposed rule statement. | -| Sprint 38 | Notifications Studio Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-38-001 | Stand up notifier CI pipelines, event bus fixtures, base dashboards for events/notifications latency. | -| Sprint 38 | Notifications Studio Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-NOTIFY-38-001 | Package notifier API/worker Helm overlays (email/chat/webhook), secrets templates, rollout guide. | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-001 | Java input normalizer (jar/war/ear/fat/jmod/jimage) with MR overlay selection. | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-002 | Module/classpath builder with duplicate & split-package detection. | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-003 | SPI scanner & provider selection with warnings. | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-004 | Reflection/TCCL heuristics emitting reason-coded edges. | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-005 | Framework config extraction (Spring, Jakarta, MicroProfile, logging, Graal configs). | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-006 | JNI/native hint detection for Java artifacts. | +| Sprint 39 | Java Analyzer Core | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-007 | Manifest/signature metadata collector (main/start/agent classes, signers). | +| Sprint 39 | Notifications Studio Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-NOTIFY-39-002 | Publish `/docs/notifications/rules.md`, `/templates.md`, `/digests.md` with imposed rule reminder. | +| Sprint 39 | Notifications Studio Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-39-002 | Add throttling/quiet-hours dashboards, digest job monitoring, and storm breaker alerts. | +| Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-NOTIFY-39-001 | Add simulation/digest CLI verbs and advanced filtering for incidents. | +| Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-NOTIFY-39-001 | Optimize digest queries and provide API for notifier to fetch unresolved policy violations/SBOM deltas. | | Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-39-001 | Implement correlation engine, throttling, quiet hours/maintenance evaluator, and incident state machine. | | Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-39-002 | Add digests generator with Findings Ledger queries and distribution (email/chat). | | Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-39-003 | Provide simulation engine and API for rule dry-run against historical events. | | Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-39-004 | Integrate quiet hours calendars and default throttles with audit logging. | -| Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-NOTIFY-39-001 | Optimize digest queries and provide API for notifier to fetch unresolved policy violations/SBOM deltas. | | Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-NOTIFY-39-001 | Surface digest scheduling, simulation, and throttle management endpoints via gateway. | -| Sprint 39 | Notifications Studio Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-NOTIFY-39-001 | Add simulation/digest CLI verbs and advanced filtering for incidents. | -| Sprint 39 | Notifications Studio Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-NOTIFY-39-002 | Publish `/docs/notifications/rules.md`, `/templates.md`, `/digests.md` with imposed rule reminder. | -| Sprint 39 | Notifications Studio Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-39-002 | Add throttling/quiet-hours dashboards, digest job monitoring, and storm breaker alerts. | +| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild | SCANNER-ANALYZERS-JAVA-21-008 | Observation writer producing entrypoints/components/edges with warnings. | +| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild, QA Guild | SCANNER-ANALYZERS-JAVA-21-009 | Fixture suite + determinism/perf benchmarks for Java analyzer. | +| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-JAVA-21-010 | Optional runtime ingestion via agent/JFR producing runtime edges. | +| Sprint 40 | Java Observation & Runtime | src/StellaOps.Scanner.Analyzers.Lang.Java/TASKS.md | TODO | Java Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-JAVA-21-011 | Package Java analyzer plug-in + Offline Kit/CLI updates. | +| Sprint 40 | Notifications Studio Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-NOTIFY-40-001 | Publish `/docs/notifications/channels.md`, `/escalations.md`, `/api.md`, `/operations/notifier-runbook.md`, `/security/notifications-hardening.md` with imposed rule lines. | +| Sprint 40 | Notifications Studio Phase 3 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-NOTIFY-40-001 | Package notifier escalations + localization deployment overlays, signed ack token rotation scripts, and rollback guidance. | +| Sprint 40 | Notifications Studio Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-40-001 | Finalize notifier dashboards/alerts (escalation failures, ack latency), chaos testing harness, and channel health monitoring. | +| Sprint 40 | Notifications Studio Phase 3 | ops/offline-kit/TASKS.md | CARRY (no scope change) | Offline Kit Guild | DEVOPS-OFFLINE-37-002 | Carry from Sprint 37: Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks. | +| Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-NOTIFY-40-001 | Enforce ack token signing/rotation, webhook allowlists, and admin-only escalation settings. | +| Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-NOTIFY-40-001 | Implement ack token redemption, escalation management, localization previews. | | Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-40-001 | Implement escalations, on-call schedules, ack bridge, PagerDuty/OpsGenie adapters, and localization bundles. | | Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-40-002 | Add CLI inbox/in-app feed channels and summary storm breaker notifications. | | Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-40-003 | Harden security: signed ack links, webhook HMAC/IP allowlists, tenant isolation fuzzing, localization fallback. | | Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-40-004 | Finalize observability (incident metrics, escalation latency) and chaos tests for channel outages. | -| Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-NOTIFY-40-001 | Enforce ack token signing/rotation, webhook allowlists, and admin-only escalation settings. | | Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-NOTIFY-40-001 | Expose escalation, localization, channel health endpoints and verification of signed links. | -| Sprint 40 | Notifications Studio Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-NOTIFY-40-001 | Implement ack token redemption, escalation management, localization previews. | -| Sprint 40 | Notifications Studio Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-NOTIFY-40-001 | Publish `/docs/notifications/channels.md`, `/escalations.md`, `/api.md`, `/operations/notifier-runbook.md`, `/security/notifications-hardening.md` with imposed rule lines. | -| Sprint 40 | Notifications Studio Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-40-001 | Finalize notifier dashboards/alerts (escalation failures, ack latency), chaos testing harness, and channel health monitoring. | -| Sprint 40 | Notifications Studio Phase 3 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-NOTIFY-40-001 | Package notifier escalations + localization deployment overlays, signed ack token rotation scripts, and rollback guidance. | -| Sprint 40 | Notifications Studio Phase 3 | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-37-001 | Bundle notifier sample configs, template/digest packs, and incident playbook into Offline Kit with integrity checks. | +| Sprint 41 | CLI Parity & Task Packs Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-41-001 | Publish `/docs/cli/overview.md`, `/cli/configuration.md`, `/cli/output-and-exit-codes.md` (with imposed rule). | +| Sprint 41 | CLI Parity & Task Packs Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-CLI-41-001 | Package CLI release artifacts (tarballs, completions, container image) with distribution docs. | +| Sprint 41 | CLI Parity & Task Packs Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CLI-41-001 | Establish CLI build pipeline (multi-platform binaries, SBOM, checksums) and parity matrix CI enforcement. | +| Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-PACKS-41-001 | Define CLI SSO scopes and Packs (`Packs.Read/Write/Run/Approve`) roles; update discovery/offline defaults. | | Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-CORE-41-001 | Implement CLI config/auth foundation, global flags, output renderer, and error/exit code mapping. | | Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PARITY-41-001 | Deliver parity command groups (`policy`, `sbom`, `vuln`, `vex`, `advisory`, `export`, `orchestrator`) with JSON/table outputs and `--explain`. | | Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PARITY-41-002 | Implement `notify`, `aoc`, `auth` command groups, idempotency keys, completions, and parity matrix export. | -| Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-41-001 | Bootstrap Task Runner service, migrations, run API, local executor, approvals pause, artifact capture. | -| Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.PacksRegistry/TASKS.md | TODO | Packs Registry Guild | PACKS-REG-41-001 | Implement packs index API, signature verification, provenance storage, and RBAC. | | Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-41-101 | Register `pack-run` job type, integrate logs/artifacts, expose pack run metadata. | -| Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-PACKS-41-001 | Define CLI SSO scopes and Packs (`Packs.Read/Write/Run/Approve`) roles; update discovery/offline defaults. | -| Sprint 41 | CLI Parity & Task Packs Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-41-001 | Publish `/docs/cli/overview.md`, `/cli/configuration.md`, `/cli/output-and-exit-codes.md` (with imposed rule). | -| Sprint 41 | CLI Parity & Task Packs Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CLI-41-001 | Establish CLI build pipeline (multi-platform binaries, SBOM, checksums) and parity matrix CI enforcement. | -| Sprint 41 | CLI Parity & Task Packs Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-CLI-41-001 | Package CLI release artifacts (tarballs, completions, container image) with distribution docs. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PARITY-41-001..002 | Close parity gaps for Notifications, Policy Studio advanced features, SBOM graph, Vuln Explorer; parity matrix green. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PACKS-42-001 | Implement Task Pack CLI commands (`pack plan/run/push/pull/verify`) with plan/simulate engine and expression sandbox. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-42-001 | Add loops, conditionals, `maxParallel`, outputs, simulation mode, policy gates in Task Runner. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.PacksRegistry/TASKS.md | TODO | Packs Registry Guild | PACKS-REG-42-001 | Support pack version lifecycle, tenant allowlists, provenance export, signature rotation. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-42-101 | Stream pack run logs via SSE/WS, expose artifact manifests, enforce pack run quotas. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-42-201 | Provide stable rationale IDs/APIs for CLI `--explain` and pack policy gates. | -| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-PACKS-42-001 | Expose snapshot/time-travel APIs for CLI offline mode and pack simulation. | +| Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.PacksRegistry/TASKS.md | TODO | Packs Registry Guild | PACKS-REG-41-001 | Implement packs index API, signature verification, provenance storage, and RBAC. | +| Sprint 41 | CLI Parity & Task Packs Phase 1 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-41-001 | Bootstrap Task Runner service, migrations, run API, local executor, approvals pause, artifact capture. | | Sprint 42 | CLI Parity & Task Packs Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-42-001 | Publish `/docs/cli/parity-matrix.md`, `/cli/commands/*.md`, `/docs/task-packs/spec.md` (imposed rule). | | Sprint 42 | CLI Parity & Task Packs Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CLI-42-001 | Add CLI golden output tests, parity diff automation, and pack run CI harness. | -| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PACKS-42-001 | Deliver advanced pack features (approvals pause/resume, remote streaming, secret injection), localization, man pages. | -| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-42-001 | Implement approvals workflow, notifications integration, remote artifact uploads, chaos resilience. | -| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.PacksRegistry/TASKS.md | TODO | Packs Registry Guild | PACKS-REG-42-001 | Enforce pack signing policies, audit trails, registry mirroring, Offline Kit support. | -| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-005, PACKS-REG-41-001 | Integrate pack run manifests into export bundles and CLI verify flows. | -| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-SVC-40-001 | Emit `pack.run.started|completed|failed` notifications with CLI deep links. | -| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-PACKS-41-001 | Enforce pack signing policies, approval RBAC, CLI token scopes for CI headless runs. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PACKS-42-001 | Implement Task Pack CLI commands (`pack plan/run/push/pull/verify`) with plan/simulate engine and expression sandbox. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PARITY-41-001..002 | Close parity gaps for Notifications, Policy Studio advanced features, SBOM graph, Vuln Explorer; parity matrix green. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-PACKS-42-001 | Expose snapshot/time-travel APIs for CLI offline mode and pack simulation. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-SVC-42-101 | Stream pack run logs via SSE/WS, expose artifact manifests, enforce pack run quotas. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.PacksRegistry/TASKS.md | TODO | Packs Registry Guild | PACKS-REG-42-001 | Support pack version lifecycle, tenant allowlists, provenance export, signature rotation. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ENGINE-42-201 | Provide stable rationale IDs/APIs for CLI `--explain` and pack policy gates. | +| Sprint 42 | CLI Parity & Task Packs Phase 2 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-42-001 | Add loops, conditionals, `maxParallel`, outputs, simulation mode, policy gates in Task Runner. | | Sprint 43 | CLI Parity & Task Packs Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-PACKS-43-001 | Publish `/docs/task-packs/authoring-guide.md`, `/registry.md`, `/runbook.md`, `/security/pack-signing-and-rbac.md`, `/operations/cli-release-and-packaging.md` (imposed rule). | | Sprint 43 | CLI Parity & Task Packs Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CLI-43-001 | Finalize multi-platform release automation, SBOM signing, parity gate enforcement, pack run chaos tests. | -| Sprint 40 | Notifications Studio Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NOTIFY-40-001 | Finalize notifier dashboards/alerts (escalation failures, ack latency), chaos testing harness, and channel health monitoring. | -| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DOCKER-44-001 | Author multi-stage Dockerfiles with non-root users, read-only FS, and health scripts for all services. | -| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DOCKER-44-002 | Generate SBOMs and cosign attestations for each image; integrate signature verification in CI. | -| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DOCKER-44-003 | Ensure `/health/*`, `/version`, `/metrics`, and capability endpoints (`merge=false`) are exposed across services. | +| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-PACKS-41-001 | Enforce pack signing policies, approval RBAC, CLI token scopes for CI headless runs. | +| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-PACKS-42-001 | Deliver advanced pack features (approvals pause/resume, remote streaming, secret injection), localization, man pages. | +| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-SVC-35-005, PACKS-REG-41-001 | Integrate pack run manifests into export bundles and CLI verify flows. | +| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.PacksRegistry/TASKS.md | TODO | Packs Registry Guild | PACKS-REG-42-001 | Enforce pack signing policies, audit trails, registry mirroring, Offline Kit support. | +| Sprint 43 | CLI Parity & Task Packs Phase 3 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-42-001 | Implement approvals workflow, notifications integration, remote artifact uploads, chaos resilience. | +| Sprint 44 | Containerized Distribution Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-44-001 | Publish install overview + Compose Quickstart docs (imposed rule). | | Sprint 44 | Containerized Distribution Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | COMPOSE-44-001 | Deliver Quickstart Compose stack with seed data and quickstart script. | | Sprint 44 | Containerized Distribution Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | COMPOSE-44-002 | Provide backup/reset scripts with guardrails and documentation. | | Sprint 44 | Containerized Distribution Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | COMPOSE-44-003 | Implement seed job and onboarding wizard toggle (`QUICKSTART_MODE`). | -| Sprint 44 | Containerized Distribution Phase 1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONTAINERS-44-001 | Expose config discovery and quickstart handling with health/version endpoints. | -| Sprint 44 | Containerized Distribution Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-44-001 | Publish install overview + Compose Quickstart docs (imposed rule). | -| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONTAINERS-44-001 | Automate multi-arch builds with SBOM/signature pipeline. | | Sprint 44 | Containerized Distribution Phase 1 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-COMPOSE-44-001 | Finalize Quickstart scripts and README. | +| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONTAINERS-44-001 | Automate multi-arch builds with SBOM/signature pipeline. | +| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DOCKER-44-001 | Author multi-stage Dockerfiles with non-root users, read-only FS, and health scripts for all services. | +| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DOCKER-44-002 | Generate SBOMs and cosign attestations for each image; integrate signature verification in CI. | +| Sprint 44 | Containerized Distribution Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DOCKER-44-003 | Ensure `/health/*`, `/version`, `/metrics`, and capability endpoints (`merge=false`) are exposed across services. | +| Sprint 44 | Containerized Distribution Phase 1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONTAINERS-44-001 | Expose config discovery and quickstart handling with health/version endpoints. | +| Sprint 45 | Containerized Distribution Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-45-001 | Publish Helm production + configuration reference docs (imposed rule). | +| Sprint 45 | Containerized Distribution Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-HELM-45-001 | Publish Helm install guide and sample values. | | Sprint 45 | Containerized Distribution Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | HELM-45-001 | Scaffold Helm chart with component toggles and pinned digests. | | Sprint 45 | Containerized Distribution Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | HELM-45-002 | Add security features (TLS, NetworkPolicy, Secrets integration). | | Sprint 45 | Containerized Distribution Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | HELM-45-003 | Implement HPA, PDB, readiness gates, and observability hooks. | -| Sprint 45 | Containerized Distribution Phase 2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONTAINERS-45-001 | Ensure readiness endpoints and config toggles support Helm deployments. | -| Sprint 45 | Containerized Distribution Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-45-001 | Publish Helm production + configuration reference docs (imposed rule). | | Sprint 45 | Containerized Distribution Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONTAINERS-45-001 | Add Compose/Helm smoke tests to CI. | -| Sprint 45 | Containerized Distribution Phase 2 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-HELM-45-001 | Publish Helm install guide and sample values. | +| Sprint 45 | Containerized Distribution Phase 2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONTAINERS-45-001 | Ensure readiness endpoints and config toggles support Helm deployments. | | Sprint 46 | Containerized Distribution Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-46-001 | Publish air-gap, supply chain, health/readiness, image catalog, console onboarding docs (imposed rule). | -| Sprint 46 | Containerized Distribution Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONTAINERS-46-001 | Build signed air-gap bundle and verify in CI. | | Sprint 46 | Containerized Distribution Phase 3 | ops/deployment/TASKS.md | TODO | Deployment Guild | DEPLOY-AIRGAP-46-001 | Provide air-gap load script and docs. | +| Sprint 46 | Containerized Distribution Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-CONTAINERS-46-001 | Build signed air-gap bundle and verify in CI. | | Sprint 46 | Containerized Distribution Phase 3 | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | OFFLINE-CONTAINERS-46-001 | Include air-gap bundle and instructions in Offline Kit. | | Sprint 46 | Containerized Distribution Phase 3 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-CONTAINERS-46-001 | Harden offline mode and document fallback behavior. | -| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-TEN-47-001 | Implement unified JWT/ODIC config, scope grammar, tenant/project claims, and JWKS caching in Authority. | -| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-TEN-47-001 | Add auth middleware (token verification, tenant activation, scope checks) and structured 403 responses. | -| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-TEN-47-001 | Ship `stella login`, `whoami`, `tenants list`, and tenant flag persistence with secure token storage. | -| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-TEN-47-001 | Integrate JWKS caching, signature verification tests, and auth regression suite into CI. | | Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | docs/TASKS.md | TODO | Docs Guild | DOCS-TEN-47-001 | Publish `/docs/security/tenancy-overview.md` and `/docs/security/scopes-and-roles.md` (imposed rule). | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-TEN-48-001 | Enforce tenant context through persistence (DB GUC, object store prefix), add request annotations, and emit audit events. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-TEN-48-001 | Add `tenant_id`/`project_id` to policy data, enable Postgres RLS, and expose rationale IDs with tenant context. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-TEN-48-001 | Partition findings by tenant/project, enable RLS, and update queries/events to include tenant context. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-TEN-48-001 | Stamp jobs with tenant/project, set DB session context, and reject jobs without context. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-TEN-48-001 | Propagate tenant/project to all steps, enforce object store prefix, and validate before execution. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-TEN-48-001 | Add tenant prefixes to manifests/artifacts, enforce scope checks, and block cross-tenant exports by default. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-TEN-48-001 | Tenant-scope notification rules, incidents, and outbound channels; update storage schemas. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-TEN-48-001 | Ensure advisory linkers operate per tenant with RLS, enforce aggregation-only capability endpoint. | -| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-TEN-48-001 | Same as above for VEX linkers; enforce capability endpoint `merge=false`. | +| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-TEN-47-001 | Integrate JWKS caching, signature verification tests, and auth regression suite into CI. | +| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-TEN-47-001 | Implement unified JWT/ODIC config, scope grammar, tenant/project claims, and JWKS caching in Authority. | +| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-TEN-47-001 | Ship `stella login`, `whoami`, `tenants list`, and tenant flag persistence with secure token storage. | +| Sprint 47 | Authority-Backed Scopes & Tenancy Phase 1 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-TEN-47-001 | Add auth middleware (token verification, tenant activation, scope checks) and structured 403 responses. | | Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | docs/TASKS.md | TODO | Docs Guild | DOCS-TEN-48-001 | Publish `/docs/operations/multi-tenancy.md`, `/docs/operations/rls-and-data-isolation.md`, `/docs/console/admin-tenants.md` (imposed rule). | | Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-TEN-48-001 | Write integration tests for RLS enforcement, tenant audit stream, and object store prefix checks. | -| Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-TEN-49-001 | Implement service accounts, delegation tokens (`act` chain), per-tenant quotas, and audit log streaming. | -| Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-TEN-49-001 | Integrate ABAC policy overlay (optional), expose audit API, and support service token minting endpoints. | -| Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-TEN-49-001 | Add service account token minting, delegation, and `--impersonate` banner/controls. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-TEN-48-001 | Ensure advisory linkers operate per tenant with RLS, enforce aggregation-only capability endpoint. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-TEN-48-001 | Same as above for VEX linkers; enforce capability endpoint `merge=false`. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-TEN-48-001 | Add tenant prefixes to manifests/artifacts, enforce scope checks, and block cross-tenant exports by default. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-TEN-48-001 | Partition findings by tenant/project, enable RLS, and update queries/events to include tenant context. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-TEN-48-001 | Tenant-scope notification rules, incidents, and outbound channels; update storage schemas. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-TEN-48-001 | Stamp jobs with tenant/project, set DB session context, and reject jobs without context. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-TEN-48-001 | Add `tenant_id`/`project_id` to policy data, enable Postgres RLS, and expose rationale IDs with tenant context. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-TEN-48-001 | Propagate tenant/project to all steps, enforce object store prefix, and validate before execution. | +| Sprint 48 | Authority-Backed Scopes & Tenancy Phase 2 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-TEN-48-001 | Enforce tenant context through persistence (DB GUC, object store prefix), add request annotations, and emit audit events. | | Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | docs/TASKS.md | TODO | Docs Guild | DOCS-TEN-49-001 | Publish `/docs/cli/authentication.md`, `/docs/api/authentication.md`, `/docs/policy/examples/abac-overlays.md`, `/docs/install/configuration-reference.md` updates (imposed rule). | | Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-TEN-49-001 | Implement audit log pipeline, monitor scope usage, chaos tests for JWKS outage, and tenant load/perf tests. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-50-001 | Bootstrap telemetry core library with structured logging, OTLP exporters, and deterministic bootstrap. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-50-002 | Deliver context propagation middleware for HTTP/gRPC/jobs/CLI carrying trace + tenant metadata. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-50-001 | Deploy default OpenTelemetry collector manifests with secure OTLP pipeline. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-50-002 | Stand up multi-tenant metrics/logs/traces backends with retention and isolation. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-50-003 | Package telemetry stack configs for offline/air-gapped installs with signatures. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OBS-50-001 | Introduce observability/timeline/evidence/attestation scopes and update discovery metadata. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-50-001 | Integrate telemetry core into gateway and emit structured traces/logs for all routes. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-50-001 | Instrument orchestrator scheduler/control APIs with telemetry core spans/logs. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-50-001 | Adopt telemetry core in Task Runner host and workers with scrubbed transcripts. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-50-001 | Wire telemetry core through ledger writer/projector for append/replay operations. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-50-001 | Replace ad-hoc logging with telemetry core across advisory ingestion/linking. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-50-001 | Adopt telemetry core in Concelier APIs and surface correlation IDs. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-50-001 | Integrate telemetry core into VEX ingestion/linking with scope metadata. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001 | Add telemetry core to VEX APIs and emit trace headers. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-50-001 | Instrument policy compile/evaluate flows with telemetry core spans/logs. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-50-001 | Enable telemetry core in export planner/workers capturing bundle metadata. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-50-001 | Propagate trace headers from CLI commands and print correlation IDs. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-OBS-50-001 | Author `/docs/observability/overview.md` with imposed rule banner and architecture context. | +| Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-TEN-49-001 | Implement service accounts, delegation tokens (`act` chain), per-tenant quotas, and audit log streaming. | +| Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-TEN-49-001 | Add service account token minting, delegation, and `--impersonate` banner/controls. | +| Sprint 49 | Authority-Backed Scopes & Tenancy Phase 3 | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-TEN-49-001 | Integrate ABAC policy overlay (optional), expose audit API, and support service token minting endpoints. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-50-001 | Add `/docs/install/telemetry-stack.md` for collector deployment and offline packaging. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | BLOCKED (2025-10-26) | Docs Guild | DOCS-OBS-50-001 | Author `/docs/observability/overview.md` with imposed rule banner and architecture context. | | Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-OBS-50-002 | Document telemetry standards (fields, scrubbing, sampling) under `/docs/observability/telemetry-standards.md`. | | Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-OBS-50-003 | Publish structured logging guide `/docs/observability/logging.md` with examples and imposed rule banner. | | Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-OBS-50-004 | Publish tracing guide `/docs/observability/tracing.md` covering context propagation and sampling. | | Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-SEC-OBS-50-001 | Update `/docs/security/redaction-and-privacy.md` for telemetry privacy controls. | -| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | docs/TASKS.md | TODO | Docs Guild | DOCS-INSTALL-50-001 | Add `/docs/install/telemetry-stack.md` for collector deployment and offline packaging. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-51-001 | Ship metrics helpers + exemplar guards for golden signals. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Security Guild | TELEMETRY-OBS-51-002 | Implement logging scrubbing and tenant debug override controls. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-OBS-50-001 | Deploy default OpenTelemetry collector manifests with secure OTLP pipeline. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | ops/devops/TASKS.md | DOING (2025-10-26) | DevOps Guild | DEVOPS-OBS-50-002 | Stand up multi-tenant metrics/logs/traces backends with retention and isolation. | +> Staging rollout plan recorded in `docs/ops/telemetry-storage.md`; waiting on Authority-issued tokens and namespace bootstrap. +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | ops/devops/TASKS.md | DONE (2025-10-26) | DevOps Guild | DEVOPS-OBS-50-003 | Package telemetry stack configs for offline/air-gapped installs with signatures. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OBS-50-001 | Introduce observability/timeline/evidence/attestation scopes and update discovery metadata. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-50-001 | Propagate trace headers from CLI commands and print correlation IDs. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-50-001 | Replace ad-hoc logging with telemetry core across advisory ingestion/linking. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-50-001 | Adopt telemetry core in Concelier APIs and surface correlation IDs. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-50-001 | Integrate telemetry core into VEX ingestion/linking with scope metadata. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-50-001 | Add telemetry core to VEX APIs and emit trace headers. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-50-001 | Enable telemetry core in export planner/workers capturing bundle metadata. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-50-001 | Wire telemetry core through ledger writer/projector for append/replay operations. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-50-001 | Instrument orchestrator scheduler/control APIs with telemetry core spans/logs. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-50-001 | Instrument policy compile/evaluate flows with telemetry core spans/logs. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-50-001 | Adopt telemetry core in Task Runner host and workers with scrubbed transcripts. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-50-001 | Bootstrap telemetry core library with structured logging, OTLP exporters, and deterministic bootstrap. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-50-002 | Deliver context propagation middleware for HTTP/gRPC/jobs/CLI carrying trace + tenant metadata. | +| Sprint 50 | Observability & Forensics Phase 1 – Baseline Telemetry | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-50-001 | Integrate telemetry core into gateway and emit structured traces/logs for all routes. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | docs/TASKS.md | TODO | Docs Guild | DOCS-OBS-51-001 | Publish `/docs/observability/metrics-and-slos.md` with alert policies. | | Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-51-001 | Deploy SLO evaluator service, dashboards, and alert routing. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OBS-51-001 | Ingest SLO burn-rate webhooks and deliver observability alerts. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-51-001 | Expose `/obs/health` and `/obs/slo` aggregations for services. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-51-001 | Publish orchestration metrics, SLOs, and burn-rate alerts. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-51-001 | Emit task runner golden-signal metrics and SLO alerts. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-51-001 | Add ledger/projector metrics dashboards and burn-rate policies. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-51-001 | Implement `stella obs top` streaming health metrics command. | | Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-51-001 | Emit ingest latency metrics + SLO thresholds for advisories. | | Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-51-001 | Provide VEX ingest metrics and SLO burn-rate automation. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-51-001 | Publish policy evaluation metrics + dashboards meeting SLO targets. | | Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-51-001 | Capture export planner/bundle latency metrics and SLOs. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-51-001 | Implement `stella obs top` streaming health metrics command. | -| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | docs/TASKS.md | TODO | Docs Guild | DOCS-OBS-51-001 | Publish `/docs/observability/metrics-and-slos.md` with alert policies. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-52-001 | Bootstrap timeline indexer service and schema with RLS scaffolding. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-52-002 | Implement event ingestion pipeline with ordering and dedupe. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-52-003 | Expose timeline query APIs with tenant filters and pagination. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Security Guild | TIMELINE-OBS-52-004 | Finalize RLS + scope enforcement and audit logging for timeline reads. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-51-001 | Add ledger/projector metrics dashboards and burn-rate policies. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OBS-51-001 | Ingest SLO burn-rate webhooks and deliver observability alerts. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-51-001 | Publish orchestration metrics, SLOs, and burn-rate alerts. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-51-001 | Publish policy evaluation metrics + dashboards meeting SLO targets. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-51-001 | Emit task runner golden-signal metrics and SLO alerts. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-51-001 | Ship metrics helpers + exemplar guards for golden signals. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Security Guild | TELEMETRY-OBS-51-002 | Implement logging scrubbing and tenant debug override controls. | +| Sprint 51 | Observability & Forensics Phase 2 – SLOs & Dashboards | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-51-001 | Expose `/obs/health` and `/obs/slo` aggregations for services. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-OBS-52-001 | Document `stella obs` CLI commands and scripting patterns. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-OBS-52-001 | Document Console observability hub and trace/log search workflows. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-OBS-52-002 | Publish Console forensics/timeline guidance with imposed rule banner. | | Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-52-001 | Configure streaming pipelines and schema validation for timeline events. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-52-001 | Provide trace/log proxy endpoints bridging to timeline + log store. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-52-001 | Emit job lifecycle timeline events with tenant/project metadata. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-52-001 | Emit pack run timeline events and dedupe logic. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-52-001 | Record ledger append/projection events into timeline stream. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-52-001 | Add `stella obs trace` + log commands correlating timeline data. | | Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-52-001 | Emit advisory ingest/link timeline events with provenance metadata. | | Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-52-001 | Provide SSE bridge for advisory timeline events. | | Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-52-001 | Emit VEX ingest/link timeline events with justification info. | | Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-52-001 | Stream VEX timeline updates to clients with tenant filters. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-52-001 | Emit policy decision timeline events with rule summaries and trace IDs. | | Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-52-001 | Publish export lifecycle events into timeline. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-52-001 | Add `stella obs trace` + log commands correlating timeline data. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-OBS-52-001 | Document Console observability hub and trace/log search workflows. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | docs/TASKS.md | TODO | Docs Guild | DOCS-CONSOLE-OBS-52-002 | Publish Console forensics/timeline guidance with imposed rule banner. | -| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-OBS-52-001 | Document `stella obs` CLI commands and scripting patterns. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-52-001 | Record ledger append/projection events into timeline stream. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-52-001 | Emit job lifecycle timeline events with tenant/project metadata. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-52-001 | Emit policy decision timeline events with rule summaries and trace IDs. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-52-001 | Emit pack run timeline events and dedupe logic. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-52-001 | Bootstrap timeline indexer service and schema with RLS scaffolding. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-52-002 | Implement event ingestion pipeline with ordering and dedupe. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-52-003 | Expose timeline query APIs with tenant filters and pagination. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Security Guild | TIMELINE-OBS-52-004 | Finalize RLS + scope enforcement and audit logging for timeline reads. | +| Sprint 52 | Observability & Forensics Phase 3 – Timeline & Decision Logs | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-52-001 | Provide trace/log proxy endpoints bridging to timeline + log store. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-FORENSICS-53-001 | Document `stella forensic` CLI workflows with sample bundles. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | docs/TASKS.md | TODO | Docs Guild | DOCS-FORENSICS-53-001 | Publish `/docs/forensics/evidence-locker.md` covering bundles, WORM, legal holds. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | docs/TASKS.md | TODO | Docs Guild | DOCS-FORENSICS-53-003 | Publish `/docs/forensics/timeline.md` with schema and query examples. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-53-001 | Provision WORM-capable storage, legal hold automation, and backup/restore scripts for evidence locker. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-FORENSICS-53-001 | Ship `stella forensic snapshot` commands invoking evidence locker. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-53-001 | Generate advisory evidence payloads (raw doc, linkset diff) for locker. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-53-001 | Add `/evidence/advisories/*` gateway endpoints consuming locker APIs. | | Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-53-001 | Bootstrap evidence locker service with schema, storage abstraction, and RLS. | | Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-53-002 | Implement bundle builders for evaluation, job, and export snapshots. | | Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-53-003 | Expose evidence APIs (create/get/verify/hold) with audit + quotas. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-53-001 | Link timeline events to evidence bundle digests and expose evidence lookup endpoint. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-53-001 | Attach job capsules + manifests to evidence locker snapshots. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-53-001 | Capture step transcripts and manifests into evidence bundles. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-53-001 | Persist evidence bundle references alongside ledger entries and expose lookup API. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-53-001 | Generate advisory evidence payloads (raw doc, linkset diff) for locker. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-53-001 | Add `/evidence/advisories/*` gateway endpoints consuming locker APIs. | | Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-53-001 | Produce VEX evidence payloads and push to locker. | | Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-53-001 | Expose `/evidence/vex/*` endpoints retrieving locker bundles. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-53-001 | Build evaluation evidence bundles (inputs, rule traces, engine version). | | Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-53-001 | Store export manifests + transcripts within evidence bundles. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-FORENSICS-53-001 | Ship `stella forensic snapshot` commands invoking evidence locker. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-53-001 | Provision WORM-capable storage, legal hold automation, and backup/restore scripts for evidence locker. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | docs/TASKS.md | TODO | Docs Guild | DOCS-FORENSICS-53-001 | Publish `/docs/forensics/evidence-locker.md` covering bundles, WORM, legal holds. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | docs/TASKS.md | TODO | Docs Guild | DOCS-FORENSICS-53-003 | Publish `/docs/forensics/timeline.md` with schema and query examples. | -| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | docs/TASKS.md | TODO | Docs Guild | DOCS-CLI-FORENSICS-53-001 | Document `stella forensic` CLI workflows with sample bundles. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-53-001 | Persist evidence bundle references alongside ledger entries and expose lookup API. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-53-001 | Attach job capsules + manifests to evidence locker snapshots. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-53-001 | Build evaluation evidence bundles (inputs, rule traces, engine version). | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-53-001 | Capture step transcripts and manifests into evidence bundles. | +| Sprint 53 | Observability & Forensics Phase 4 – Evidence Locker | src/StellaOps.TimelineIndexer/TASKS.md | TODO | Timeline Indexer Guild | TIMELINE-OBS-53-001 | Link timeline events to evidence bundle digests and expose evidence lookup endpoint. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | docs/TASKS.md | TODO | Docs Guild | DOCS-FORENSICS-53-002 | Publish `/docs/forensics/provenance-attestation.md` covering signing + verification. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-54-001 | Manage provenance signing infrastructure (KMS keys, timestamp authority) and CI verification. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-FORENSICS-54-001 | Implement `stella forensic verify` command verifying bundles + signatures. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-FORENSICS-54-002 | Add `stella forensic attest show` command with signer/timestamp details. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-54-001 | Sign advisory batches with DSSE attestations and expose verification. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-54-001 | Add `/attestations/advisories/*` endpoints surfacing verification metadata. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-54-001 | Attach DSSE signing/timestamping to evidence bundles and emit timeline hooks. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-54-002 | Provide bundle packaging + offline verification fixtures. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-54-001 | Produce VEX batch attestations linking to timeline/ledger. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-54-001 | Expose `/attestations/vex/*` endpoints with verification summaries. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-54-001 | Produce export attestation manifests and CLI verification hooks. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-54-001 | Produce DSSE attestations for jobs and surface verification endpoint. | +| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-54-001 | Generate DSSE attestations for policy evaluations and expose verification API. | | Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Provenance.Attestation/TASKS.md | TODO | Provenance Guild | PROV-OBS-53-001 | Implement DSSE/SLSA models with deterministic serializer + test vectors. | | Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Provenance.Attestation/TASKS.md | TODO | Provenance Guild | PROV-OBS-53-002 | Build signer abstraction (cosign/KMS/offline) with policy enforcement. | | Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Provenance.Attestation/TASKS.md | TODO | Provenance Guild | PROV-OBS-54-001 | Deliver verification library validating DSSE signatures + Merkle roots. | | Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Provenance.Attestation/TASKS.md | TODO | Provenance Guild, DevEx/CLI Guild | PROV-OBS-54-002 | Package provenance verification tool for CLI integration and offline use. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-54-001 | Attach DSSE signing/timestamping to evidence bundles and emit timeline hooks. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-54-002 | Provide bundle packaging + offline verification fixtures. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-54-001 | Produce DSSE attestations for jobs and surface verification endpoint. | | Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-54-001 | Generate pack run attestations and link to timeline/evidence. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-54-001 | Sign advisory batches with DSSE attestations and expose verification. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-54-001 | Add `/attestations/advisories/*` endpoints surfacing verification metadata. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-54-001 | Produce VEX batch attestations linking to timeline/ledger. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-54-001 | Expose `/attestations/vex/*` endpoints with verification summaries. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-54-001 | Generate DSSE attestations for policy evaluations and expose verification API. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-54-001 | Produce export attestation manifests and CLI verification hooks. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-FORENSICS-54-001 | Implement `stella forensic verify` command verifying bundles + signatures. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-FORENSICS-54-002 | Add `stella forensic attest show` command with signer/timestamp details. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | docs/TASKS.md | TODO | Docs Guild | DOCS-FORENSICS-53-002 | Publish `/docs/forensics/provenance-attestation.md` covering signing + verification. | -| Sprint 54 | Observability & Forensics Phase 5 – Provenance & Verification | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-54-001 | Manage provenance signing infrastructure (KMS keys, timestamp authority) and CI verification. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-55-001 | Implement incident mode sampling toggle API with activation audit trail. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-55-001 | Deliver `/obs/incident-mode` control endpoints with audit + retention previews. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-55-001 | Increase telemetry + evidence capture during incident mode and emit activation events. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-55-001 | Capture extra debug data + notifications for incident mode runs. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-55-001 | Extend retention and diagnostics capture during incident mode. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNBOOK-55-001 | Publish `/docs/runbooks/incidents.md` covering activation, escalation, and verification checklist. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-55-001 | Automate incident mode activation via SLO alerts, retention override management, and reset job. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OBS-55-001 | Enforce `obs:incident` scope with fresh-auth requirement and audit export for toggles. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-55-001 | Ship `stella obs incident-mode` commands with safeguards and audit logging. | | Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OBS-55-001 | Increase sampling and raw payload retention under incident mode with redaction guards. | | Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OBS-55-001 | Provide incident mode toggle endpoints and propagate to services. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-55-001 | Extend evidence retention + activation events for incident windows. | | Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OBS-55-001 | Enable incident sampling + retention overrides for VEX pipelines. | | Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OBS-55-001 | Add incident mode APIs for VEX services with audit + guardrails. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-55-001 | Capture full rule traces + retention bump on incident activation with timeline events. | | Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OBS-55-001 | Increase export telemetry + debug retention during incident mode and emit events. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-55-001 | Extend evidence retention + activation events for incident windows. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-55-001 | Ship `stella obs incident-mode` commands with safeguards and audit logging. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OBS-55-001 | Automate incident mode activation via SLO alerts, retention override management, and reset job. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OBS-55-001 | Extend retention and diagnostics capture during incident mode. | | Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OBS-55-001 | Send incident mode start/stop notifications with quick links to evidence/timeline. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OBS-55-001 | Enforce `obs:incident` scope with fresh-auth requirement and audit export for toggles. | -| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNBOOK-55-001 | Publish `/docs/runbooks/incidents.md` covering activation, escalation, and verification checklist. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Controller/TASKS.md | TODO | AirGap Controller Guild | AIRGAP-CTL-56-001 | Implement sealing state machine, persistence, and RBAC scopes for air-gapped status. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Controller/TASKS.md | TODO | AirGap Controller Guild | AIRGAP-CTL-56-002 | Expose seal/status APIs with policy hash validation and staleness placeholders. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-56-001 | Ship `EgressPolicy` facade with sealed/unsealed enforcement and remediation errors. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-56-002 | Deliver Roslyn analyzer blocking raw HTTP clients; wire into CI. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-56-001 | Publish deny-all egress policies and verification script for sealed environments. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-56-002 | Provide bundle staging/import scripts for air-gapped object stores. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-56-003 | Build Bootstrap Pack pipeline bundling images/charts with checksums. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-56-001 | (Carry) Extend telemetry core with sealed-mode hooks before integration. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-56-001 | Extend telemetry core usage for sealed-mode status surfaces (seal/unseal dashboards, drift signals). | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-56-001 | Implement mirror create/verify and airgap verify commands. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Mirror.Creator/TASKS.md | TODO | Mirror Creator Guild | MIRROR-CRT-56-001 | Build deterministic bundle assembler (advisories/vex/policy). | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-56-001 | Implement DSSE/TUF/Merkle verification helpers. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-56-002 | Enforce root rotation policy for bundles. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-AIRGAP-56-001 | Add mirror ingestion adapters preserving source metadata. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-AIRGAP-56-001 | Add VEX mirror ingestion adapters. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-56-001 | Accept policy packs from bundles with provenance tracking. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-56-001 | Extend export center to build mirror bundles. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-56-001 | Enforce sealed-mode plan validation for network calls. | -| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-56-001 | Validate jobs against sealed-mode restrictions. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OBS-55-001 | Increase telemetry + evidence capture during incident mode and emit activation events. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-OBS-55-001 | Capture full rule traces + retention bump on incident activation with timeline events. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OBS-55-001 | Capture extra debug data + notifications for incident mode runs. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-55-001 | Implement incident mode sampling toggle API with activation audit trail. | +| Sprint 55 | Observability & Forensics Phase 6 – Incident Mode | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-55-001 | Deliver `/obs/incident-mode` control endpoints with audit + retention previews. | | Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-56-001 | Publish `/docs/airgap/overview.md`. | | Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-56-002 | Document sealing and egress controls. | | Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-56-003 | Publish mirror bundles guide. | | Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-56-004 | Publish bootstrap pack guide. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-56-001 | Publish deny-all egress policies and verification script for sealed environments. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-56-002 | Provide bundle staging/import scripts for air-gapped object stores. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-56-003 | Build Bootstrap Pack pipeline bundling images/charts with checksums. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Controller/TASKS.md | TODO | AirGap Controller Guild | AIRGAP-CTL-56-001 | Implement sealing state machine, persistence, and RBAC scopes for air-gapped status. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Controller/TASKS.md | TODO | AirGap Controller Guild | AIRGAP-CTL-56-002 | Expose seal/status APIs with policy hash validation and staleness placeholders. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-56-001 | Implement DSSE/TUF/Merkle verification helpers. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-56-002 | Enforce root rotation policy for bundles. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-56-001 | Ship `EgressPolicy` facade with sealed/unsealed enforcement and remediation errors. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-56-002 | Deliver Roslyn analyzer blocking raw HTTP clients; wire into CI. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-56-001 | Implement mirror create/verify and airgap verify commands. | | Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-OBS-50-001 | Ensure telemetry propagation for sealed logging. | - -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-57-001 | Implement bundle catalog with RLS + migrations. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-57-002 | Load artifacts into object store with checksum verification. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Mirror.Creator/TASKS.md | TODO | Mirror Creator Guild | MIRROR-CRT-57-001 | Add OCI image support to mirror bundles. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Mirror.Creator/TASKS.md | TODO | Mirror Creator Guild | MIRROR-CRT-57-002 | Embed signed time anchors in bundles. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Time/TASKS.md | TODO | AirGap Time Guild | AIRGAP-TIME-57-001 | Parse signed time tokens and expose normalized anchors. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-57-001 | Adopt EgressPolicy in core services. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-57-002 | Enforce Task Runner job plan validation. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-56-002 | Provide bundle ingestion helper steps. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-56-002 | Integrate sealing status + staleness into scheduling. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-57-001 | Complete airgap import CLI with diff preview. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-57-002 | Ship seal/status CLI commands. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-56-002 | Deliver bootstrap pack artifacts. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-56-001 | Lock notifications to enclave-safe channels. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-57-001 | Automate mirror bundle creation with approvals. | -| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-57-002 | Run sealed-mode CI suite enforcing zero egress. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-AIRGAP-56-001 | Add mirror ingestion adapters preserving source metadata. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-AIRGAP-56-001 | Add VEX mirror ingestion adapters. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-56-001 | Extend export center to build mirror bundles. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Mirror.Creator/TASKS.md | TODO | Mirror Creator Guild | MIRROR-CRT-56-001 | Build deterministic bundle assembler (advisories/vex/policy). | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-56-001 | Validate jobs against sealed-mode restrictions. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-56-001 | Accept policy packs from bundles with provenance tracking. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-56-001 | Enforce sealed-mode plan validation for network calls. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Telemetry.Core/TASKS.md | TODO | Observability Guild | TELEMETRY-OBS-56-001 | (Carry) Extend telemetry core with sealed-mode hooks before integration. | +| Sprint 56 | Air-Gapped Mode Phase 1 – Sealing Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OBS-56-001 | Extend telemetry core usage for sealed-mode status surfaces (seal/unseal dashboards, drift signals). | | Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-57-001 | Publish staleness/time doc. | | Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-57-002 | Publish console airgap doc. | | Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-57-003 | Publish CLI airgap doc. | | Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-57-004 | Publish airgap operations runbook. | - -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Time/TASKS.md | TODO | AirGap Time Guild | AIRGAP-TIME-58-001 | Compute drift/staleness metrics and surface via controller status. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Time/TASKS.md | TODO | AirGap Time Guild | AIRGAP-TIME-58-002 | Emit notifications/events for staleness budgets. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Controller/TASKS.md | TODO | AirGap Controller Guild | AIRGAP-CTL-58-001 | Persist time anchor data and expose drift metrics. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-58-001 | Disable remote observability exporters in sealed mode. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-58-002 | Add CLI sealed-mode guard. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-58-001 | Ship portable evidence export helper. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-58-001 | Capture import job evidence transcripts. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-58-001 | Link import/export jobs to timeline/evidence. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-AIRGAP-57-002 | Annotate advisories with staleness metadata. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-AIRGAP-57-002 | Annotate VEX statements with staleness metadata. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-57-002 | Show degradation fallback info in explain traces. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-57-001 | Notify on drift/staleness thresholds. | -| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-57-001 | Add portable evidence export integration. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-57-001 | Automate mirror bundle creation with approvals. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-AIRGAP-57-002 | Run sealed-mode CI suite enforcing zero egress. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-57-001 | Implement bundle catalog with RLS + migrations. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Importer/TASKS.md | TODO | AirGap Importer Guild | AIRGAP-IMP-57-002 | Load artifacts into object store with checksum verification. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-57-001 | Adopt EgressPolicy in core services. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-57-002 | Enforce Task Runner job plan validation. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.AirGap.Time/TASKS.md | TODO | AirGap Time Guild | AIRGAP-TIME-57-001 | Parse signed time tokens and expose normalized anchors. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-57-001 | Complete airgap import CLI with diff preview. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-57-002 | Ship seal/status CLI commands. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-56-002 | Deliver bootstrap pack artifacts. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Mirror.Creator/TASKS.md | TODO | Mirror Creator Guild | MIRROR-CRT-57-001 | Add OCI image support to mirror bundles. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Mirror.Creator/TASKS.md | TODO | Mirror Creator Guild | MIRROR-CRT-57-002 | Embed signed time anchors in bundles. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-56-001 | Lock notifications to enclave-safe channels. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-56-002 | Integrate sealing status + staleness into scheduling. | +| Sprint 57 | Air-Gapped Mode Phase 2 – Mirror Bundles & Imports | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-56-002 | Provide bundle ingestion helper steps. | | Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-58-001 | Publish degradation matrix doc. | | Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-58-002 | Update trust & signing doc for DSSE/TUF roots. | | Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-58-003 | Publish developer airgap contracts doc. | | Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-58-004 | Document portable evidence workflows. | - -| Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-57-001 | Block execution when seal state mismatched; emit timeline events. | -| Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-57-001 | Automate mirror bundle job scheduling with audit provenance. | -| Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-57-001 | Enforce sealed-mode guardrails inside evaluation engine. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Controller/TASKS.md | TODO | AirGap Controller Guild | AIRGAP-CTL-58-001 | Persist time anchor data and expose drift metrics. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-58-001 | Disable remote observability exporters in sealed mode. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Policy/TASKS.md | TODO | AirGap Policy Guild | AIRGAP-POL-58-002 | Add CLI sealed-mode guard. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Time/TASKS.md | TODO | AirGap Time Guild | AIRGAP-TIME-58-001 | Compute drift/staleness metrics and surface via controller status. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.AirGap.Time/TASKS.md | TODO | AirGap Time Guild | AIRGAP-TIME-58-002 | Emit notifications/events for staleness budgets. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-58-001 | Ship portable evidence export helper. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-AIRGAP-57-002 | Annotate advisories with staleness metadata. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-AIRGAP-57-002 | Annotate VEX statements with staleness metadata. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-57-001 | Add portable evidence export integration. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-57-001 | Notify on drift/staleness thresholds. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-58-001 | Link import/export jobs to timeline/evidence. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-57-002 | Show degradation fallback info in explain traces. | +| Sprint 58 | Air-Gapped Mode Phase 3 – Staleness & Enforcement | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-58-001 | Capture import job evidence transcripts. | | Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-AIRGAP-57-001 | Map sealed-mode violations to standard errors. | | Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-AIRGAP-57-001 | Map sealed-mode violations to standard errors. | | Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-AIRGAP-58-001 | Emit notifications/timeline for bundle readiness. | | Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-AIRGAP-56-002 | Enforce staleness thresholds for findings exports. | | Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-58-001 | Notify on portable evidence exports. | - -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-55-001 | Extend retention + portable evidence exports for sealed environments. | -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-58-001 | Finalize portable evidence CLI workflow with verification. | -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-AIRGAP-57-001 | Link findings to portable evidence bundles. | -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-58-001 | Notify on stale policy packs and guide remediation. | -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-AIRGAP-58-001 | Emit timeline events for bundle imports. | -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-AIRGAP-58-001 | Emit timeline events for VEX bundle imports. | -| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-58-001 | (Carry) Portable evidence notifications. | +| Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-AIRGAP-57-001 | Automate mirror bundle job scheduling with audit provenance. | +| Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-57-001 | Enforce sealed-mode guardrails inside evaluation engine. | +| Sprint 59 | Air-Gapped Mode Phase 4 – Deterministic Jobs & Enforcement | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-AIRGAP-57-001 | Block execution when seal state mismatched; emit timeline events. | | Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-58-004 | Document portable evidence workflows. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-61-001 | Scaffold per-service OpenAPI skeletons with shared components. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-61-002 | Build aggregate composer and integrate into CI. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-AIRGAP-58-001 | Finalize portable evidence CLI workflow with verification. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-AIRGAP-58-001 | Emit timeline events for bundle imports. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.EvidenceLocker/TASKS.md | TODO | Evidence Locker Guild | EVID-OBS-60-001 | Deliver portable evidence export flow for sealed environments with checksum manifest and offline verification script. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-AIRGAP-58-001 | Emit timeline events for VEX bundle imports. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-AIRGAP-57-001 | Link findings to portable evidence bundles. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-AIRGAP-58-001 | (Carry) Portable evidence notifications. | +| Sprint 60 | Air-Gapped Mode Phase 5 – Evidence Portability & UX | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-AIRGAP-58-001 | Notify on stale policy packs and guide remediation. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-61-001 | Publish `/docs/api/overview.md`. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-61-002 | Publish `/docs/api/conventions.md`. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-61-003 | Publish `/docs/api/versioning.md`. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OAS-61-001 | Add OAS lint/validation/diff stages to CI. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-61-001 | Configure lint rules and CI enforcement. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-61-002 | Enforce example coverage in CI. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-OAS-61-001 | Add OAS lint/validation/diff stages to CI. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-61-001 | Implement gateway discovery endpoint. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-61-002 | Standardize error envelope across gateway. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OAS-61-001 | Extend Orchestrator spec coverage. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OAS-61-002 | Provide orchestrator discovery endpoint. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-61-001 | Document Task Runner APIs in OAS. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-61-002 | Expose Task Runner discovery endpoint. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-61-001 | Scaffold per-service OpenAPI skeletons with shared components. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-61-002 | Build aggregate composer and integrate into CI. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OAS-61-001 | Document Authority authentication APIs in OAS. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OAS-61-002 | Provide Authority discovery endpoint. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OAS-61-001 | Update advisory OAS coverage. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OAS-61-002 | Populate advisory examples. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OAS-61-001 | Implement Concelier discovery endpoint. | @@ -1040,211 +981,195 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OAS-61-002 | Provide VEX examples. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OAS-61-001 | Implement discovery endpoint. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OAS-61-002 | Migrate errors to standard envelope. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-61-001 | Expand Findings Ledger spec coverage. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-61-002 | Provide ledger discovery endpoint. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OAS-61-001 | Update Exporter spec coverage. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OAS-61-002 | Implement Exporter discovery endpoint. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-61-001 | Expand Findings Ledger spec coverage. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-61-002 | Provide ledger discovery endpoint. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-61-001 | Update notifier spec coverage. | | Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-61-002 | Implement notifier discovery endpoint. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OAS-61-001 | Document Authority authentication APIs in OAS. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OAS-61-002 | Provide Authority discovery endpoint. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-61-001 | Publish `/docs/api/overview.md`. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-61-002 | Publish `/docs/api/conventions.md`. | -| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-61-003 | Publish `/docs/api/versioning.md`. | - -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-62-001 | Populate examples for top endpoints. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-62-001 | Implement compatibility diff tool. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-62-001 | Build static generator with nav/search. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-62-002 | Add schema viewer, examples, version selector. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OAS-61-001 | Extend Orchestrator spec coverage. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OAS-61-002 | Provide orchestrator discovery endpoint. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-61-001 | Document Task Runner APIs in OAS. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-61-002 | Expose Task Runner discovery endpoint. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-61-001 | Implement gateway discovery endpoint. | +| Sprint 61 | SDKs & OpenAPI Phase 1 – Contract Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-61-002 | Standardize error envelope across gateway. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-CONTRIB-62-001 | Publish API contracts contributing guide. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-DEVPORT-62-001 | Document dev portal publishing. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-OAS-62-001 | Deploy `/docs/api/reference/` generated site. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-SDK-62-001 | Publish SDK overview + language guides. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-DEVPORT-62-001 | Document dev portal publishing. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-CONTRIB-62-001 | Publish API contracts contributing guide. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-TEST-62-001 | Publish contract testing doc. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-SEC-62-001 | Update auth scopes documentation. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-62-001 | Establish generator framework. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-62-002 | Implement shared post-processing helpers. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-62-002 | Integrate schema diagrams/examples. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-62-001 | Provide SDK examples for pack runs. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OAS-62-001 | Ensure SDK streaming helpers for exports. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-62-001 | Provide SDK examples for notifier APIs. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OAS-62-001 | Add SDK smoke tests for advisory APIs. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OAS-62-001 | Add SDK tests for VEX APIs. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-62-001 | Provide SDK tests for ledger APIs. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | docs/TASKS.md | TODO | Docs Guild | DOCS-TEST-62-001 | Publish contract testing doc. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-62-001 | Implement compatibility diff tool. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-62-001 | Populate examples for top endpoints. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OAS-62-001 | Provide SDK auth helpers/tests. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-62-001 | Align pagination/idempotency behaviors. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OAS-62-001 | Add advisory API examples. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OAS-62-001 | Provide VEX API examples. | -| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-62-001 | Publish notifier examples. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SDK-62-001 | Migrate CLI to official SDK. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SDK-62-002 | Update CLI error handling for new envelope. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OAS-62-001 | Add SDK smoke tests for advisory APIs. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-OAS-62-001 | Add advisory API examples. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-62-001 | Build static generator with nav/search. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-62-002 | Add schema viewer, examples, version selector. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OAS-62-001 | Add SDK tests for VEX APIs. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Excititor WebService Guild | EXCITITOR-WEB-OAS-62-001 | Provide VEX API examples. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OAS-62-001 | Ensure SDK streaming helpers for exports. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-62-001 | Provide SDK tests for ledger APIs. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-62-001 | Provide SDK examples for notifier APIs. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-62-001 | Establish generator framework. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-62-002 | Implement shared post-processing helpers. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-62-001 | Provide SDK examples for pack runs. | +| Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-62-001 | Align pagination/idempotency behaviors. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | test/contract/TASKS.md | TODO | Contract Testing Guild | CONTR-62-001 | Generate mock server fixtures. | | Sprint 62 | SDKs & OpenAPI Phase 2 – Examples & Portal | test/contract/TASKS.md | TODO | Contract Testing Guild | CONTR-62-002 | Integrate mock server into CI. | - +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | docs/TASKS.md | TODO | Docs Guild | DOCS-TEST-62-001 | (Carry) ensure contract testing doc final. | +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-63-001 | Integrate compatibility diff gating. | +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-63-001 | Compatibility diff support. | +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-63-002 | Define discovery schema metadata. | +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SDK-63-001 | Add CLI spec download command. | +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-63-001 | Add Try-It console. | +| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-63-002 | Embed SDK snippets/quick starts. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-63-001 | Release TypeScript SDK alpha. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-63-002 | Release Python SDK alpha. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-63-003 | Release Go SDK alpha. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-63-004 | Release Java SDK alpha. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-63-001 | Configure SDK release pipelines. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-63-002 | Automate changelogs from OAS diffs. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-63-001 | Add Try-It console. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-63-002 | Embed SDK snippets/quick starts. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | test/contract/TASKS.md | TODO | Contract Testing Guild | CONTR-63-001 | Build replay harness for drift detection. | | Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | test/contract/TASKS.md | TODO | Contract Testing Guild | CONTR-63-002 | Emit contract testing metrics. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SDK-63-001 | Add CLI spec download command. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-63-001 | Integrate compatibility diff gating. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-63-001 | Compatibility diff support. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | src/StellaOps.Api.OpenApi/TASKS.md | TODO | API Contracts Guild | OAS-63-002 | Define discovery schema metadata. | -| Sprint 63 | SDKs & OpenAPI Phase 3 – SDK Alpha & Try-It | docs/TASKS.md | TODO | Docs Guild | DOCS-TEST-62-001 | (Carry) ensure contract testing doc final. | - -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-64-001 | Migrate CLI to SDK. | -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-64-002 | Integrate SDKs into Console. | -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-64-001 | Hook SDK releases to Notifications. | -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-64-002 | Produce devportal offline bundle. | +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-DEVPORT-64-001 | Document devportal offline usage. | +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-DEVPORT-63-001 | Automate developer portal pipeline. | +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-DEVPORT-64-001 | Schedule offline bundle builds. | | Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-64-001 | Offline portal build. | | Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.DevPortal.Site/TASKS.md | TODO | Developer Portal Guild | DEVPORT-64-002 | Add accessibility/performance checks. | | Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.ExportCenter.DevPortalOffline/TASKS.md | TODO | DevPortal Offline Guild | DVOFF-64-001 | Implement devportal offline export job. | | Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.ExportCenter.DevPortalOffline/TASKS.md | TODO | DevPortal Offline Guild | DVOFF-64-002 | Provide verification CLI. | -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-DEVPORT-63-001 | Automate developer portal pipeline. | -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-DEVPORT-64-001 | Schedule offline bundle builds. | -| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-DEVPORT-64-001 | Document devportal offline usage. | - +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-64-001 | Migrate CLI to SDK. | +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Generator/TASKS.md | TODO | SDK Generator Guild | SDKGEN-64-002 | Integrate SDKs into Console. | +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-64-001 | Hook SDK releases to Notifications. | +| Sprint 64 | SDKs & OpenAPI Phase 4 – Harden & Offline Bundles | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-64-002 | Produce devportal offline bundle. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-DEVPORT-64-001 | (Carry) ensure offline doc published; update as necessary. | | Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-63-001 | (Carry) compatibility gating monitoring. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Api.Governance/TASKS.md | TODO | API Governance Guild | APIGOV-63-001 | (Reinforce) notifications integration for deprecations. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-64-001 | Production rollout of notifications feed. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-63-001 | Emit deprecation notifications. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-63-001 | Implement deprecation headers in gateway. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OAS-63-001 | Add orchestrator deprecation headers. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-63-001 | Add Task Runner deprecation headers. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OAS-63-001 | Deprecation metadata for Concelier APIs. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OAS-63-001 | Deprecation metadata for VEX APIs. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-63-001 | Deprecation headers for ledger APIs. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OAS-63-001 | Deprecation headers for exporter APIs. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-63-001 | Deprecation headers for notifier APIs. | | Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-OAS-63-001 | Deprecation headers for auth endpoints. | | Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-SDK-64-001 | SDK update awareness command. | -| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | docs/TASKS.md | TODO | Docs Guild | DOCS-AIRGAP-DEVPORT-64-001 | (Carry) ensure offline doc published; update as necessary. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-66-001 | Deliver RiskProfile schema + validators. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-66-002 | Implement inheritance/merge and hashing. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-66-001 | Scaffold risk engine queue/worker/registry. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-66-002 | Implement transforms/gates/contribution calculator. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-66-003 | Integrate schema validation into Policy Engine. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-RISK-66-004 | Extend Policy libraries for RiskProfile handling. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-66-001 | Add risk scoring columns/indexes. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-66-002 | Implement deterministic scoring upserts. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-RISK-66-001 | Expose CVSS/KEV provider data. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-RISK-66-002 | Provide fix availability signals. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-RISK-66-001 | Supply VEX gating data to risk engine. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-RISK-66-002 | Provide reachability inputs. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-66-001 | Expose risk API routing in gateway. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-66-002 | Handle explainability downloads. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-66-001 | Implement CLI profile management commands. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-66-002 | Implement CLI simulation command. | -| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-66-001 | Create risk severity alert templates. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-OAS-63-001 | Deprecation metadata for Concelier APIs. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-OAS-63-001 | Deprecation metadata for VEX APIs. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-OAS-63-001 | Deprecation headers for exporter APIs. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-OAS-63-001 | Deprecation headers for ledger APIs. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-OAS-63-001 | Emit deprecation notifications. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Orchestrator/TASKS.md | TODO | Orchestrator Service Guild | ORCH-OAS-63-001 | Add orchestrator deprecation headers. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Sdk.Release/TASKS.md | TODO | SDK Release Guild | SDKREL-64-001 | Production rollout of notifications feed. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.TaskRunner/TASKS.md | TODO | Task Runner Guild | TASKRUN-OAS-63-001 | Add Task Runner deprecation headers. | +| Sprint 65 | SDKs & OpenAPI Phase 5 – Deprecation & Notifications | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-OAS-63-001 | Implement deprecation headers in gateway. | | Sprint 66 | Risk Profiles Phase 1 – Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-66-001 | Publish `/docs/risk/overview.md`. | | Sprint 66 | Risk Profiles Phase 1 – Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-66-002 | Publish `/docs/risk/profiles.md`. | | Sprint 66 | Risk Profiles Phase 1 – Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-66-003 | Publish `/docs/risk/factors.md`. | | Sprint 66 | Risk Profiles Phase 1 – Foundations | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-66-004 | Publish `/docs/risk/formulas.md`. | - -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-67-001 | Integrate CVSS/KEV providers. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-67-002 | Integrate VEX gate provider. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-67-003 | Add fix availability/criticality/exposure providers. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-67-001 | Integrate profiles into policy store lifecycle. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-67-002 | Publish schema endpoint + validation tooling. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-67-001 | Enqueue scoring on new findings. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-67-002 | Deliver profile lifecycle APIs. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-RISK-67-003 | Provide simulation orchestration APIs. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-67-001 | Notify on profile publish/deprecate. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-RISK-67-001 | Add source consensus metrics. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-RISK-67-001 | Add VEX explainability metadata. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-67-001 | Provide risk status endpoint. | -| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-67-001 | Provide risk results query command. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-66-001 | Implement CLI profile management commands. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-66-002 | Implement CLI simulation command. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-RISK-66-001 | Expose CVSS/KEV provider data. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-RISK-66-002 | Provide fix availability signals. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-RISK-66-001 | Supply VEX gating data to risk engine. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-RISK-66-002 | Provide reachability inputs. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-66-001 | Add risk scoring columns/indexes. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-66-002 | Implement deterministic scoring upserts. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-66-001 | Create risk severity alert templates. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-66-003 | Integrate schema validation into Policy Engine. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-66-001 | Deliver RiskProfile schema + validators. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-66-002 | Implement inheritance/merge and hashing. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-RISK-66-004 | Extend Policy libraries for RiskProfile handling. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-66-001 | Scaffold risk engine queue/worker/registry. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-66-002 | Implement transforms/gates/contribution calculator. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-66-001 | Expose risk API routing in gateway. | +| Sprint 66 | Risk Profiles Phase 1 – Foundations | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-66-002 | Handle explainability downloads. | | Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-001 | Publish explainability doc. | | Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-002 | Publish risk API doc. | | Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-003 | Publish console risk UI doc. | | Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-004 | Publish CLI risk doc. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-67-001 | Provide risk results query command. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Concelier.Core/TASKS.md | TODO | Concelier Core Guild | CONCELIER-RISK-67-001 | Add source consensus metrics. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Excititor.Core/TASKS.md | TODO | Excititor Core Guild | EXCITITOR-RISK-67-001 | Add VEX explainability metadata. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-67-001 | Notify on profile publish/deprecate. | | Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-68-001 | (Prep) risk routing settings seeds. | - -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-68-001 | Persist scoring results & explanations. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-68-002 | Expose jobs/results/explanations APIs. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-67-001 | Provide scored findings query API. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-68-001 | Enable scored findings export. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-68-001 | Ship simulation API endpoint. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-RISK-68-002 | Support profile export/import. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-68-001 | Configure risk notification routing UI/logic. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-68-001 | Emit severity transition events via gateway. | -| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-68-001 | Add risk bundle verification command. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-67-001 | Enqueue scoring on new findings. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-67-002 | Deliver profile lifecycle APIs. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-67-001 | Integrate profiles into policy store lifecycle. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy.RiskProfile/TASKS.md | TODO | Risk Profile Schema Guild | POLICY-RISK-67-002 | Publish schema endpoint + validation tooling. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-RISK-67-003 | Provide simulation orchestration APIs. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-67-001 | Integrate CVSS/KEV providers. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-67-002 | Integrate VEX gate provider. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-67-003 | Add fix availability/criticality/exposure providers. | +| Sprint 67 | Risk Profiles Phase 2 – Providers & Lifecycle | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-67-001 | Provide risk status endpoint. | | Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-68-001 | Publish risk bundle doc. | | Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-68-002 | Update AOC invariants doc. | - -| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-69-001 | Implement simulation mode. | -| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-69-002 | Add telemetry/metrics dashboards. | -| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-66-001 | (Completion) finalize severity alert templates. | -| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-RISK-69-002 | Enable simulation report exports. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-68-001 | Add risk bundle verification command. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-67-001 | Provide scored findings query API. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-68-001 | Enable scored findings export. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-68-001 | Configure risk notification routing UI/logic. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-RISK-68-001 | Ship simulation API endpoint. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-RISK-68-002 | Support profile export/import. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-68-001 | Persist scoring results & explanations. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-68-002 | Expose jobs/results/explanations APIs. | +| Sprint 68 | Risk Profiles Phase 3 – APIs & Ledger | src/StellaOps.Web/TASKS.md | TODO | BE-Base Platform Guild | WEB-RISK-68-001 | Emit severity transition events via gateway. | +| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-001..004 | (Carry) ensure docs updated from simulation release. | | Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.ExportCenter.RiskBundles/TASKS.md | TODO | Risk Bundle Export Guild | RISK-BUNDLE-69-001 | Build risk bundle. | | Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.ExportCenter.RiskBundles/TASKS.md | TODO | Risk Bundle Export Guild | RISK-BUNDLE-69-002 | Integrate bundle into pipelines. | -| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-001..004 | (Carry) ensure docs updated from simulation release. | - -| Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-70-001 | Support offline provider bundles. | -| Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-70-002 | Integrate runtime/reachability providers. | +| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-RISK-69-002 | Enable simulation report exports. | +| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-66-001 | (Completion) finalize severity alert templates. | +| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-69-001 | Implement simulation mode. | +| Sprint 69 | Risk Profiles Phase 4 – Simulation & Reporting | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-69-002 | Add telemetry/metrics dashboards. | +| Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-68-001 | (Carry) finalize risk bundle doc after verification CLI. | | Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.ExportCenter.RiskBundles/TASKS.md | TODO | Risk Bundle Export Guild | RISK-BUNDLE-70-001 | Provide bundle verification CLI. | | Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.ExportCenter.RiskBundles/TASKS.md | TODO | Risk Bundle Export Guild | RISK-BUNDLE-70-002 | Publish documentation. | | Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.ExportCenter/TASKS.md | TODO | Exporter Service Guild | EXPORT-RISK-70-001 | Integrate risk bundle into offline kit. | | Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-68-001 | Finalize risk alert routing UI. | -| Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-68-001 | (Carry) finalize risk bundle doc after verification CLI. | - -| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-69-002 | Optimize performance, cache, and incremental scoring; validate SLOs. | -| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-69-001 | Finalize dashboards and alerts for scoring latency. | -| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-66-001..68-001 | Harden CLI commands with integration tests and error handling. | -| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-68-001 | Tune routing/quiet hour dedupe for risk alerts. | +| Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-70-001 | Support offline provider bundles. | +| Sprint 70 | Risk Profiles Phase 5 – Air-Gap & Advanced Factors | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-70-002 | Integrate runtime/reachability providers. | | Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | docs/TASKS.md | TODO | Docs Guild | DOCS-RISK-67-001..68-002 | Final editorial pass on risk documentation set. | +| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI Guild | CLI-RISK-66-001..68-001 | Harden CLI commands with integration tests and error handling. | +| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.Findings.Ledger/TASKS.md | TODO | Findings Ledger Guild | LEDGER-RISK-69-001 | Finalize dashboards and alerts for scoring latency. | +| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-RISK-68-001 | Tune routing/quiet hour dedupe for risk alerts. | +| Sprint 71 | Risk Profiles Phase 6 – Quality & Performance | src/StellaOps.RiskEngine/TASKS.md | TODO | Risk Engine Guild | RISK-ENGINE-69-002 | Optimize performance, cache, and incremental scoring; validate SLOs. | +| Sprint 72 | Attestor Console Phase 1 – Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ATTEST-73-001 | (Prep) align CI secrets for Attestor service. | | Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Attestor.Envelope/TASKS.md | TODO | Envelope Guild | ATTEST-ENVELOPE-72-001 | Implement DSSE canonicalization and hashing helpers. | | Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Attestor.Envelope/TASKS.md | TODO | Envelope Guild | ATTEST-ENVELOPE-72-002 | Support compact/expanded output and detached payloads. | | Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Attestor.Types/TASKS.md | TODO | Attestation Payloads Guild | ATTEST-TYPES-72-001 | Draft schemas for all attestation payload types. | | Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Attestor.Types/TASKS.md | TODO | Attestation Payloads Guild | ATTEST-TYPES-72-002 | Generate models/validators from schemas. | -| Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Cryptography.Kms/TASKS.md | TODO | KMS Guild | KMS-72-001 | Implement KMS interface + file driver. | | Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-72-001 | Scaffold attestor service skeleton. | | Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-72-002 | Implement attestation store + storage integration. | -| Sprint 72 | Attestor Console Phase 1 – Foundations | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ATTEST-73-001 | (Prep) align CI secrets for Attestor service. | - -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Cryptography.Kms/TASKS.md | TODO | KMS Guild | KMS-72-002 | CLI support for key import/export. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor.Envelope/TASKS.md | TODO | Envelope Guild | ATTEST-ENVELOPE-73-001 | Add signing/verification helpers with KMS integration. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor.Types/TASKS.md | TODO | Attestation Payloads Guild | ATTEST-TYPES-73-001 | Create golden payload fixtures. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-73-001 | Ship signing endpoint. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-73-002 | Ship verification pipeline and reports. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-73-003 | Implement list/fetch APIs. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ATTEST-73-001 | Implement VerificationPolicy lifecycle. | -| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ATTEST-73-002 | Surface policies in Policy Studio. | +| Sprint 72 | Attestor Console Phase 1 – Foundations | src/StellaOps.Cryptography.Kms/TASKS.md | TODO | KMS Guild | KMS-72-001 | Implement KMS interface + file driver. | | Sprint 73 | Attestor CLI Phase 2 – Signing & Policies | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild | CLI-ATTEST-73-001 | Implement `stella attest sign` (payload selection, subject digest, key reference, output format) using official SDK transport. | | Sprint 73 | Attestor CLI Phase 2 – Signing & Policies | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild | CLI-ATTEST-73-002 | Implement `stella attest verify` with policy selection, explainability output, and JSON/table formatting. | | Sprint 73 | Attestor Console Phase 2 – Signing & Policies | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-73-001 | Publish attestor overview. | | Sprint 73 | Attestor Console Phase 2 – Signing & Policies | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-73-002 | Publish payload docs. | | Sprint 73 | Attestor Console Phase 2 – Signing & Policies | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-73-003 | Publish policies doc. | | Sprint 73 | Attestor Console Phase 2 – Signing & Policies | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-73-004 | Publish workflows doc. | - +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor.Envelope/TASKS.md | TODO | Envelope Guild | ATTEST-ENVELOPE-73-001 | Add signing/verification helpers with KMS integration. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor.Types/TASKS.md | TODO | Attestation Payloads Guild | ATTEST-TYPES-73-001 | Create golden payload fixtures. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-73-001 | Ship signing endpoint. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-73-002 | Ship verification pipeline and reports. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-73-003 | Implement list/fetch APIs. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Cryptography.Kms/TASKS.md | TODO | KMS Guild | KMS-72-002 | CLI support for key import/export. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ATTEST-73-001 | Implement VerificationPolicy lifecycle. | +| Sprint 73 | Attestor Console Phase 2 – Signing & Policies | src/StellaOps.Policy.Engine/TASKS.md | TODO | Policy Guild | POLICY-ATTEST-73-002 | Surface policies in Policy Studio. | +| Sprint 74 | Attestor CLI Phase 3 – Transparency & Chain of Custody | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild | CLI-ATTEST-74-001 | Implement `stella attest list` with filters (subject, type, issuer, scope) and pagination. | +| Sprint 74 | Attestor CLI Phase 3 – Transparency & Chain of Custody | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild | CLI-ATTEST-74-002 | Implement `stella attest fetch` to download envelopes and payloads to disk. | +| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-001 | Publish keys & issuers doc. | +| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-002 | Publish transparency doc. | +| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-003 | Publish console attestor UI doc. | +| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-004 | Publish CLI attest doc. | +| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ATTEST-74-001 | Deploy transparency witness infra. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Attestor.Envelope/TASKS.md | TODO | Envelope Guild | ATTEST-ENVELOPE-73-002 | Run fuzz tests for envelope handling. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Attestor.Verify/TASKS.md | TODO | Verification Guild | ATTEST-VERIFY-74-001 | Add telemetry for verification pipeline. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Attestor.Verify/TASKS.md | TODO | Verification Guild | ATTEST-VERIFY-74-002 | Document verification explainability. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-74-001 | Integrate transparency witness client. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-74-002 | Implement bulk verification worker. | -| Sprint 74 | Attestor CLI Phase 3 – Transparency & Chain of Custody | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild | CLI-ATTEST-74-001 | Implement `stella attest list` with filters (subject, type, issuer, scope) and pagination. | -| Sprint 74 | Attestor CLI Phase 3 – Transparency & Chain of Custody | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild | CLI-ATTEST-74-002 | Implement `stella attest fetch` to download envelopes and payloads to disk. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.ExportCenter.AttestationBundles/TASKS.md | TODO | Attestation Bundle Guild | EXPORT-ATTEST-74-001 | Build attestation bundle export job. | -| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ATTEST-74-001 | Deploy transparency witness infra. | -| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-001 | Publish keys & issuers doc. | -| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-002 | Publish transparency doc. | -| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-003 | Publish console attestor UI doc. | -| Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-74-004 | Publish CLI attest doc. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-ATTEST-74-001 | Add verification/key notifications. | | Sprint 74 | Attestor Console Phase 3 – Transparency & Chain of Custody | src/StellaOps.Notifier/TASKS.md | TODO | Notifications Service Guild | NOTIFY-ATTEST-74-002 | Notify key rotation/revocation. | - -| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-75-001 | Support attestation bundle export/import for air gap. | -| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-75-002 | Harden APIs (rate limits, fuzz tests, threat model actions). | -| Sprint 75 | Attestor CLI Phase 4 – Air Gap & Bulk | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild, KMS Guild | CLI-ATTEST-75-001 | Implement `stella attest key create|import|rotate|revoke` commands. | | Sprint 75 | Attestor CLI Phase 4 – Air Gap & Bulk | src/StellaOps.Cli/TASKS.md | TODO | CLI Attestor Guild, Export Guild | CLI-ATTEST-75-002 | Add support for building/verifying attestation bundles in CLI. | -| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.ExportCenter.AttestationBundles/TASKS.md | TODO | Attestation Bundle Guild | EXPORT-ATTEST-75-001 | CLI bundle verify/import. | -| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.ExportCenter.AttestationBundles/TASKS.md | TODO | Attestation Bundle Guild | EXPORT-ATTEST-75-002 | Document attestor airgap workflow. | | Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-75-001 | Publish attestor airgap doc. | | Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | docs/TASKS.md | TODO | Docs Guild | DOCS-ATTEST-75-002 | Update AOC invariants for attestations. | | Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ATTEST-74-002 | Integrate bundle builds into release/offline pipelines. | | Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-ATTEST-75-001 | Dashboards/alerts for attestor metrics. | +| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-75-001 | Support attestation bundle export/import for air gap. | +| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Service Guild | ATTESTOR-75-002 | Harden APIs (rate limits, fuzz tests, threat model actions). | +| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.ExportCenter.AttestationBundles/TASKS.md | TODO | Attestation Bundle Guild | EXPORT-ATTEST-75-001 | CLI bundle verify/import. | +| Sprint 75 | Attestor Console Phase 4 – Air Gap & Bulk | src/StellaOps.ExportCenter.AttestationBundles/TASKS.md | TODO | Attestation Bundle Guild | EXPORT-ATTEST-75-002 | Document attestor airgap workflow. | diff --git a/TODOS.md b/TODOS.md deleted file mode 100644 index 3986dc99..00000000 --- a/TODOS.md +++ /dev/null @@ -1,5 +0,0 @@ -1. Concelier must never merge, just aggregate. The advisories must be linked, but their severity will be considered through policies from the Web part of the scanner. -2. Same for Excititor -3. Why Web and UI? -4. Consider all of functionality there is. Is the Cli for all of it? Is there a Web for all of it? Do we have Policy editor? Does the policy support VEX application rules? Advisories application Rules? Do have SBOM graphs explorer? Do we have good SBOM vulnerability explorer? Do we have advisory using AI? -5. Do we have build docker containers that are downloadable? page describign how to download and install? \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md index 0125c880..cf22949f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -5,20 +5,34 @@ This directory contains deterministic deployment bundles for the core Stella Ops ## Structure - `releases/` – canonical release manifests (edge, stable, airgap) used to source image digests. -- `compose/` – Docker Compose bundles for dev/stage/airgap targets plus `.env` seed files. -- `compose/docker-compose.mirror.yaml` – managed mirror bundle for `*.stella-ops.org` with gateway cache and multi-tenant auth. -- `helm/stellaops/` – multi-profile Helm chart with values files for dev/stage/airgap. -- `tools/validate-profiles.sh` – helper that runs `docker compose config` and `helm lint/template` for every profile. +- `compose/` – Docker Compose bundles for dev/stage/airgap targets plus `.env` seed files. +- `compose/docker-compose.mirror.yaml` – managed mirror bundle for `*.stella-ops.org` with gateway cache and multi-tenant auth. +- `compose/docker-compose.telemetry.yaml` – optional OpenTelemetry collector overlay (mutual TLS, OTLP pipelines). +- `compose/docker-compose.telemetry-storage.yaml` – optional Prometheus/Tempo/Loki stack for observability backends. +- `helm/stellaops/` – multi-profile Helm chart with values files for dev/stage/airgap. +- `telemetry/` – shared OpenTelemetry collector configuration and certificate artefacts (generated via tooling). +- `tools/validate-profiles.sh` – helper that runs `docker compose config` and `helm lint/template` for every profile. ## Workflow 1. Update or add a release manifest under `releases/` with the new digests. 2. Mirror the digests into the Compose and Helm profiles that correspond to that channel. -3. Run `deploy/tools/validate-profiles.sh` (requires Docker CLI and Helm) to ensure the bundles lint and template cleanly. -4. Commit the change alongside any documentation updates (e.g. install guide cross-links). - +3. Run `deploy/tools/validate-profiles.sh` (requires Docker CLI and Helm) to ensure the bundles lint and template cleanly. +4. If telemetry ingest is required for the release, generate development certificates using + `./ops/devops/telemetry/generate_dev_tls.sh` and run the collector smoke test with + `python ./ops/devops/telemetry/smoke_otel_collector.py` to verify the OTLP endpoints. +5. Commit the change alongside any documentation updates (e.g. install guide cross-links). + Maintaining the digest linkage keeps offline/air-gapped installs reproducible and avoids tag drift between environments. +### Additional tooling + +- `deploy/tools/check-channel-alignment.py` – verifies that Helm/Compose profiles reference the exact images listed in a release manifest. Run it for each channel before promoting a release. +- `ops/devops/telemetry/generate_dev_tls.sh` – produces local CA/server/client certificates for Compose-based collector testing. +- `ops/devops/telemetry/smoke_otel_collector.py` – sends OTLP traffic and asserts the collector accepted traces, metrics, and logs. +- `ops/devops/telemetry/package_offline_bundle.py` – packages telemetry assets (config/Helm/Compose) into a signed tarball for air-gapped installs. +- `docs/ops/deployment-upgrade-runbook.md` – end-to-end instructions for upgrade, rollback, and channel promotion workflows (Helm + Compose). + ## CI smoke checks The `.gitea/workflows/build-test-deploy.yml` pipeline includes a `notify-smoke` stage that validates scanner event propagation after staging deployments. Configure the following repository secrets (or environment-level secrets) so the job can connect to Redis and the Notify API: diff --git a/deploy/compose/README.md b/deploy/compose/README.md index 73a3ef46..384a1c71 100644 --- a/deploy/compose/README.md +++ b/deploy/compose/README.md @@ -7,21 +7,52 @@ These Compose bundles ship the minimum services required to exercise the scanner | Path | Purpose | | ---- | ------- | | `docker-compose.dev.yaml` | Edge/nightly stack tuned for laptops and iterative work. | -| `docker-compose.stage.yaml` | Stable channel stack mirroring pre-production clusters. | -| `docker-compose.airgap.yaml` | Stable stack with air-gapped defaults (no outbound hostnames). | -| `docker-compose.mirror.yaml` | Managed mirror topology for `*.stella-ops.org` distribution (Concelier + Excititor + CDN gateway). | -| `env/*.env.example` | Seed `.env` files that document required secrets and ports per profile. | +| `docker-compose.stage.yaml` | Stable channel stack mirroring pre-production clusters. | +| `docker-compose.prod.yaml` | Production cutover stack with front-door network hand-off and Notify events enabled. | +| `docker-compose.airgap.yaml` | Stable stack with air-gapped defaults (no outbound hostnames). | +| `docker-compose.mirror.yaml` | Managed mirror topology for `*.stella-ops.org` distribution (Concelier + Excititor + CDN gateway). | +| `docker-compose.telemetry.yaml` | Optional OpenTelemetry collector overlay (mutual TLS, OTLP ingest endpoints). | +| `docker-compose.telemetry-storage.yaml` | Prometheus/Tempo/Loki storage overlay with multi-tenant defaults. | +| `env/*.env.example` | Seed `.env` files that document required secrets and ports per profile. | ## Usage -```bash -cp env/dev.env.example dev.env -docker compose --env-file dev.env -f docker-compose.dev.yaml config -docker compose --env-file dev.env -f docker-compose.dev.yaml up -d -``` - +```bash +cp env/dev.env.example dev.env +docker compose --env-file dev.env -f docker-compose.dev.yaml config +docker compose --env-file dev.env -f docker-compose.dev.yaml up -d +``` + The stage and airgap variants behave the same way—swap the file names accordingly. All profiles expose 443/8443 for the UI and REST APIs, and they share a `stellaops` Docker network scoped to the compose project. +> **Graph Explorer reminder:** If you enable Cartographer or Graph API containers alongside these profiles, update `etc/authority.yaml` so the `cartographer-service` client is marked with `properties.serviceIdentity: "cartographer"` and carries a tenant hint. The Authority host now refuses `graph:write` tokens without that marker, so apply the configuration change before rolling out the updated images. + +### Telemetry collector overlay + +The OpenTelemetry collector overlay is optional and can be layered on top of any profile: + +```bash +./ops/devops/telemetry/generate_dev_tls.sh +docker compose -f docker-compose.telemetry.yaml up -d +python ../../ops/devops/telemetry/smoke_otel_collector.py --host localhost +docker compose -f docker-compose.telemetry-storage.yaml up -d +``` + +The generator script creates a development CA plus server/client certificates under +`deploy/telemetry/certs/`. The smoke test sends OTLP/HTTP payloads using the generated +client certificate and asserts the collector reports accepted traces, metrics, and logs. +The storage overlay starts Prometheus, Tempo, and Loki with multitenancy enabled so you +can validate the end-to-end pipeline before promoting changes to staging. Adjust the +configs in `deploy/telemetry/storage/` before running in production. +Mount the same certificates when running workloads so the collector can enforce mutual TLS. + +For production cutovers copy `env/prod.env.example` to `prod.env`, update the secret placeholders, and create the external network expected by the profile: + +```bash +docker network create stellaops_frontdoor +docker compose --env-file prod.env -f docker-compose.prod.yaml config +``` + ### Scanner event stream settings Scanner WebService can emit signed `scanner.report.*` events to Redis Streams when `SCANNER__EVENTS__ENABLED=true`. Each profile ships environment placeholders you can override in the `.env` file: @@ -35,6 +66,10 @@ Scanner WebService can emit signed `scanner.report.*` events to Redis Streams wh Helm values mirror the same knobs under each service’s `env` map (see `deploy/helm/stellaops/values-*.yaml`). +### Front-door network hand-off + +`docker-compose.prod.yaml` adds a `frontdoor` network so operators can attach Traefik, Envoy, or an on-prem load balancer that terminates TLS. Override `FRONTDOOR_NETWORK` in `prod.env` if your reverse proxy uses a different bridge name. Attach only the externally reachable services (Authority, Signer, Attestor, Concelier, Scanner Web, Notify Web, UI) to that network—internal infrastructure (Mongo, MinIO, RustFS, NATS) stays on the private `stellaops` network. + ### Updating to a new release 1. Import the new manifest into `deploy/releases/` (see `deploy/README.md`). diff --git a/deploy/compose/docker-compose.prod.yaml b/deploy/compose/docker-compose.prod.yaml new file mode 100644 index 00000000..a326fc12 --- /dev/null +++ b/deploy/compose/docker-compose.prod.yaml @@ -0,0 +1,237 @@ +x-release-labels: &release-labels + com.stellaops.release.version: "2025.09.2" + com.stellaops.release.channel: "stable" + com.stellaops.profile: "prod" + +networks: + stellaops: + driver: bridge + frontdoor: + external: true + name: ${FRONTDOOR_NETWORK:-stellaops_frontdoor} + +volumes: + mongo-data: + minio-data: + rustfs-data: + concelier-jobs: + nats-data: + +services: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + command: ["mongod", "--bind_ip_all"] + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: "${MONGO_INITDB_ROOT_USERNAME}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_INITDB_ROOT_PASSWORD}" + volumes: + - mongo-data:/data/db + networks: + - stellaops + labels: *release-labels + + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data", "--console-address", ":9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}" + volumes: + - minio-data:/data + ports: + - "${MINIO_CONSOLE_PORT:-9001}:9001" + networks: + - stellaops + labels: *release-labels + + rustfs: + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"] + restart: unless-stopped + environment: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumes: + - rustfs-data:/data + ports: + - "${RUSTFS_HTTP_PORT:-8080}:8080" + networks: + - stellaops + labels: *release-labels + + nats: + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + command: + - "-js" + - "-sd" + - /data + restart: unless-stopped + ports: + - "${NATS_CLIENT_PORT:-4222}:4222" + volumes: + - nats-data:/data + networks: + - stellaops + labels: *release-labels + + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + restart: unless-stopped + depends_on: + - mongo + environment: + STELLAOPS_AUTHORITY__ISSUER: "${AUTHORITY_ISSUER}" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + volumes: + - ../../etc/authority.yaml:/etc/authority.yaml:ro + - ../../etc/authority.plugins:/app/etc/authority.plugins:ro + ports: + - "${AUTHORITY_PORT:-8440}:8440" + networks: + - stellaops + - frontdoor + labels: *release-labels + + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + restart: unless-stopped + depends_on: + - authority + environment: + SIGNER__AUTHORITY__BASEURL: "https://authority:8440" + SIGNER__POE__INTROSPECTURL: "${SIGNER_POE_INTROSPECT_URL}" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${SIGNER_PORT:-8441}:8441" + networks: + - stellaops + - frontdoor + labels: *release-labels + + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + restart: unless-stopped + depends_on: + - signer + environment: + ATTESTOR__SIGNER__BASEURL: "https://signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${ATTESTOR_PORT:-8442}:8442" + networks: + - stellaops + - frontdoor + labels: *release-labels + + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + restart: unless-stopped + depends_on: + - mongo + - minio + environment: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + CONCELIER__AUTHORITY__BASEURL: "https://authority:8440" + volumes: + - concelier-jobs:/var/lib/concelier/jobs + ports: + - "${CONCELIER_PORT:-8445}:8445" + networks: + - stellaops + - frontdoor + labels: *release-labels + + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + restart: unless-stopped + depends_on: + - concelier + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-true}" + SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" + SCANNER__EVENTS__DSN: "${SCANNER_EVENTS_DSN:-}" + SCANNER__EVENTS__STREAM: "${SCANNER_EVENTS_STREAM:-stella.events}" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "${SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS:-5}" + SCANNER__EVENTS__MAXSTREAMLENGTH: "${SCANNER_EVENTS_MAX_STREAM_LENGTH:-10000}" + ports: + - "${SCANNER_WEB_PORT:-8444}:8444" + networks: + - stellaops + - frontdoor + labels: *release-labels + + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + restart: unless-stopped + depends_on: + - scanner-web + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + networks: + - stellaops + labels: *release-labels + + notify-web: + image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.09.2} + restart: unless-stopped + depends_on: + - mongo + - authority + environment: + DOTNET_ENVIRONMENT: Production + volumes: + - ../../etc/notify.prod.yaml:/app/etc/notify.yaml:ro + ports: + - "${NOTIFY_WEB_PORT:-8446}:8446" + networks: + - stellaops + - frontdoor + labels: *release-labels + + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + restart: unless-stopped + depends_on: + - concelier + environment: + EXCITITOR__CONCELIER__BASEURL: "https://concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + networks: + - stellaops + labels: *release-labels + + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + restart: unless-stopped + depends_on: + - scanner-web + environment: + STELLAOPS_UI__BACKEND__BASEURL: "https://scanner-web:8444" + ports: + - "${UI_PORT:-8443}:8443" + networks: + - stellaops + - frontdoor + labels: *release-labels diff --git a/deploy/compose/docker-compose.telemetry-storage.yaml b/deploy/compose/docker-compose.telemetry-storage.yaml new file mode 100644 index 00000000..cb9462f6 --- /dev/null +++ b/deploy/compose/docker-compose.telemetry-storage.yaml @@ -0,0 +1,57 @@ +version: "3.9" + +services: + prometheus: + image: prom/prometheus:v2.53.0 + container_name: stellaops-prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yaml" + volumes: + - ../telemetry/storage/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - prometheus-data:/prometheus + - ../telemetry/certs:/etc/telemetry/tls:ro + - ../telemetry/storage/auth:/etc/telemetry/auth:ro + environment: + PROMETHEUS_COLLECTOR_TARGET: stellaops-otel-collector:9464 + ports: + - "9090:9090" + depends_on: + - tempo + - loki + + tempo: + image: grafana/tempo:2.5.0 + container_name: stellaops-tempo + command: + - "-config.file=/etc/tempo/tempo.yaml" + volumes: + - ../telemetry/storage/tempo.yaml:/etc/tempo/tempo.yaml:ro + - ../telemetry/storage/tenants/tempo-overrides.yaml:/etc/telemetry/tenants/tempo-overrides.yaml:ro + - ../telemetry/certs:/etc/telemetry/tls:ro + - tempo-data:/var/tempo + ports: + - "3200:3200" + environment: + TEMPO_ZONE: docker + + loki: + image: grafana/loki:3.1.0 + container_name: stellaops-loki + command: + - "-config.file=/etc/loki/loki.yaml" + volumes: + - ../telemetry/storage/loki.yaml:/etc/loki/loki.yaml:ro + - ../telemetry/storage/tenants/loki-overrides.yaml:/etc/telemetry/tenants/loki-overrides.yaml:ro + - ../telemetry/certs:/etc/telemetry/tls:ro + - loki-data:/var/loki + ports: + - "3100:3100" + +volumes: + prometheus-data: + tempo-data: + loki-data: + +networks: + default: + name: stellaops-telemetry diff --git a/deploy/compose/docker-compose.telemetry.yaml b/deploy/compose/docker-compose.telemetry.yaml new file mode 100644 index 00000000..c94b6ac4 --- /dev/null +++ b/deploy/compose/docker-compose.telemetry.yaml @@ -0,0 +1,34 @@ +version: "3.9" + +services: + otel-collector: + image: otel/opentelemetry-collector:0.105.0 + container_name: stellaops-otel-collector + command: + - "--config=/etc/otel-collector/config.yaml" + environment: + STELLAOPS_OTEL_TLS_CERT: /etc/otel-collector/tls/collector.crt + STELLAOPS_OTEL_TLS_KEY: /etc/otel-collector/tls/collector.key + STELLAOPS_OTEL_TLS_CA: /etc/otel-collector/tls/ca.crt + STELLAOPS_OTEL_PROMETHEUS_ENDPOINT: 0.0.0.0:9464 + STELLAOPS_OTEL_REQUIRE_CLIENT_CERT: "true" + STELLAOPS_TENANT_ID: dev + volumes: + - ../telemetry/otel-collector-config.yaml:/etc/otel-collector/config.yaml:ro + - ../telemetry/certs:/etc/otel-collector/tls:ro + ports: + - "4317:4317" # OTLP gRPC (mTLS) + - "4318:4318" # OTLP HTTP (mTLS) + - "9464:9464" # Prometheus exporter (mTLS) + - "13133:13133" # Health check + - "1777:1777" # pprof + healthcheck: + test: ["CMD", "curl", "-fsk", "--cert", "/etc/otel-collector/tls/client.crt", "--key", "/etc/otel-collector/tls/client.key", "--cacert", "/etc/otel-collector/tls/ca.crt", "https://localhost:13133/healthz"] + interval: 30s + start_period: 15s + timeout: 5s + retries: 3 + +networks: + default: + name: stellaops-telemetry diff --git a/deploy/compose/env/prod.env.example b/deploy/compose/env/prod.env.example new file mode 100644 index 00000000..d912ab05 --- /dev/null +++ b/deploy/compose/env/prod.env.example @@ -0,0 +1,29 @@ +# Substitutions for docker-compose.prod.yaml +# ⚠️ Replace all placeholder secrets with values sourced from your secret manager. +MONGO_INITDB_ROOT_USERNAME=stellaops-prod +MONGO_INITDB_ROOT_PASSWORD=REPLACE_WITH_STRONG_PASSWORD +MINIO_ROOT_USER=stellaops-prod +MINIO_ROOT_PASSWORD=REPLACE_WITH_STRONG_PASSWORD +# Expose the MinIO console only to trusted operator networks. +MINIO_CONSOLE_PORT=39001 +RUSTFS_HTTP_PORT=8080 +AUTHORITY_ISSUER=https://authority.prod.stella-ops.org +AUTHORITY_PORT=8440 +SIGNER_POE_INTROSPECT_URL=https://licensing.prod.stella-ops.org/introspect +SIGNER_PORT=8441 +ATTESTOR_PORT=8442 +CONCELIER_PORT=8445 +SCANNER_WEB_PORT=8444 +UI_PORT=8443 +NATS_CLIENT_PORT=4222 +SCANNER_QUEUE_BROKER=nats://nats:4222 +# `true` enables signed scanner events for Notify ingestion. +SCANNER_EVENTS_ENABLED=true +SCANNER_EVENTS_DRIVER=redis +# Leave SCANNER_EVENTS_DSN empty to inherit the Redis queue DSN when SCANNER_QUEUE_BROKER uses redis://. +SCANNER_EVENTS_DSN= +SCANNER_EVENTS_STREAM=stella.events +SCANNER_EVENTS_PUBLISH_TIMEOUT_SECONDS=5 +SCANNER_EVENTS_MAX_STREAM_LENGTH=10000 +# External reverse proxy (Traefik, Envoy, etc.) that terminates TLS. +FRONTDOOR_NETWORK=stellaops_frontdoor diff --git a/deploy/helm/stellaops/files/otel-collector-config.yaml b/deploy/helm/stellaops/files/otel-collector-config.yaml new file mode 100644 index 00000000..d5d0167e --- /dev/null +++ b/deploy/helm/stellaops/files/otel-collector-config.yaml @@ -0,0 +1,64 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set} + key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set} + client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set} + require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true} + http: + endpoint: 0.0.0.0:4318 + tls: + cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set} + key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set} + client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set} + require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true} + +processors: + attributes/tenant-tag: + actions: + - key: tenant.id + action: insert + value: ${STELLAOPS_TENANT_ID:unknown} + batch: + send_batch_size: 1024 + timeout: 5s + +exporters: + logging: + verbosity: normal + prometheus: + endpoint: ${STELLAOPS_OTEL_PROMETHEUS_ENDPOINT:0.0.0.0:9464} + enable_open_metrics: true + metric_expiration: 5m + tls: + cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set} + key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set} + client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set} + +extensions: + health_check: + endpoint: ${STELLAOPS_OTEL_HEALTH_ENDPOINT:0.0.0.0:13133} + pprof: + endpoint: ${STELLAOPS_OTEL_PPROF_ENDPOINT:0.0.0.0:1777} + +service: + telemetry: + logs: + level: ${STELLAOPS_OTEL_LOG_LEVEL:info} + extensions: [health_check, pprof] + pipelines: + traces: + receivers: [otlp] + processors: [attributes/tenant-tag, batch] + exporters: [logging] + metrics: + receivers: [otlp] + processors: [attributes/tenant-tag, batch] + exporters: [logging, prometheus] + logs: + receivers: [otlp] + processors: [attributes/tenant-tag, batch] + exporters: [logging] diff --git a/deploy/helm/stellaops/templates/_helpers.tpl b/deploy/helm/stellaops/templates/_helpers.tpl index aa727d83..635956f4 100644 --- a/deploy/helm/stellaops/templates/_helpers.tpl +++ b/deploy/helm/stellaops/templates/_helpers.tpl @@ -1,6 +1,18 @@ {{- define "stellaops.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- end -}} + +{{- define "stellaops.telemetryCollector.config" -}} +{{- if .Values.telemetry.collector.config }} +{{ tpl .Values.telemetry.collector.config . }} +{{- else }} +{{ tpl (.Files.Get "files/otel-collector-config.yaml") . }} +{{- end }} +{{- end -}} + +{{- define "stellaops.telemetryCollector.fullname" -}} +{{- printf "%s-otel-collector" (include "stellaops.name" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} {{- define "stellaops.fullname" -}} {{- $name := default .root.Chart.Name .root.Values.fullnameOverride -}} diff --git a/deploy/helm/stellaops/templates/otel-collector.yaml b/deploy/helm/stellaops/templates/otel-collector.yaml new file mode 100644 index 00000000..f4f10f34 --- /dev/null +++ b/deploy/helm/stellaops/templates/otel-collector.yaml @@ -0,0 +1,121 @@ +{{- if .Values.telemetry.collector.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "stellaops.telemetryCollector.fullname" . }} + labels: + {{- include "stellaops.labels" (dict "root" . "name" "otel-collector" "svc" (dict "class" "telemetry")) | nindent 4 }} +data: + config.yaml: | +{{ include "stellaops.telemetryCollector.config" . | indent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "stellaops.telemetryCollector.fullname" . }} + labels: + {{- include "stellaops.labels" (dict "root" . "name" "otel-collector" "svc" (dict "class" "telemetry")) | nindent 4 }} +spec: + replicas: {{ .Values.telemetry.collector.replicas | default 1 }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "stellaops.name" . | quote }} + app.kubernetes.io/component: "otel-collector" + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "stellaops.name" . | quote }} + app.kubernetes.io/component: "otel-collector" + stellaops.profile: {{ .Values.global.profile | quote }} + spec: + containers: + - name: otel-collector + image: {{ .Values.telemetry.collector.image | default "otel/opentelemetry-collector:0.105.0" | quote }} + args: + - "--config=/etc/otel/config.yaml" + ports: + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + - name: metrics + containerPort: 9464 + - name: health + containerPort: 13133 + - name: pprof + containerPort: 1777 + env: + - name: STELLAOPS_OTEL_TLS_CERT + value: {{ .Values.telemetry.collector.tls.certPath | default "/etc/otel/tls/tls.crt" | quote }} + - name: STELLAOPS_OTEL_TLS_KEY + value: {{ .Values.telemetry.collector.tls.keyPath | default "/etc/otel/tls/tls.key" | quote }} + - name: STELLAOPS_OTEL_TLS_CA + value: {{ .Values.telemetry.collector.tls.caPath | default "/etc/otel/tls/ca.crt" | quote }} + - name: STELLAOPS_OTEL_PROMETHEUS_ENDPOINT + value: {{ .Values.telemetry.collector.prometheusEndpoint | default "0.0.0.0:9464" | quote }} + - name: STELLAOPS_OTEL_REQUIRE_CLIENT_CERT + value: {{ .Values.telemetry.collector.requireClientCert | default true | quote }} + - name: STELLAOPS_TENANT_ID + value: {{ .Values.telemetry.collector.defaultTenant | default "unknown" | quote }} + - name: STELLAOPS_OTEL_LOG_LEVEL + value: {{ .Values.telemetry.collector.logLevel | default "info" | quote }} + volumeMounts: + - name: config + mountPath: /etc/otel/config.yaml + subPath: config.yaml + readOnly: true + - name: tls + mountPath: /etc/otel/tls + readOnly: true + livenessProbe: + httpGet: + scheme: HTTPS + port: health + path: /healthz + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + scheme: HTTPS + port: health + path: /healthz + initialDelaySeconds: 5 + periodSeconds: 15 +{{- with .Values.telemetry.collector.resources }} + resources: +{{ toYaml . | indent 12 }} +{{- end }} + volumes: + - name: config + configMap: + name: {{ include "stellaops.telemetryCollector.fullname" . }} + - name: tls + secret: + secretName: {{ .Values.telemetry.collector.tls.secretName | required "telemetry.collector.tls.secretName is required" }} +{{- if .Values.telemetry.collector.tls.items }} + items: +{{ toYaml .Values.telemetry.collector.tls.items | indent 14 }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "stellaops.telemetryCollector.fullname" . }} + labels: + {{- include "stellaops.labels" (dict "root" . "name" "otel-collector" "svc" (dict "class" "telemetry")) | nindent 4 }} +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: {{ include "stellaops.name" . | quote }} + app.kubernetes.io/component: "otel-collector" + ports: + - name: otlp-grpc + port: {{ .Values.telemetry.collector.service.grpcPort | default 4317 }} + targetPort: otlp-grpc + - name: otlp-http + port: {{ .Values.telemetry.collector.service.httpPort | default 4318 }} + targetPort: otlp-http + - name: metrics + port: {{ .Values.telemetry.collector.service.metricsPort | default 9464 }} + targetPort: metrics +{{- end }} diff --git a/deploy/helm/stellaops/values-dev.yaml b/deploy/helm/stellaops/values-dev.yaml index a27d75d3..eb99fc83 100644 --- a/deploy/helm/stellaops/values-dev.yaml +++ b/deploy/helm/stellaops/values-dev.yaml @@ -1,15 +1,22 @@ -global: - profile: dev - release: - version: "2025.10.0-edge" - channel: edge - manifestSha256: "822f82987529ea38d2321dbdd2ef6874a4062a117116a20861c26a8df1807beb" - image: - pullPolicy: IfNotPresent - labels: - stellaops.io/channel: edge - -configMaps: +global: + profile: dev + release: + version: "2025.10.0-edge" + channel: edge + manifestSha256: "822f82987529ea38d2321dbdd2ef6874a4062a117116a20861c26a8df1807beb" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: edge + +telemetry: + collector: + enabled: true + defaultTenant: dev + tls: + secretName: stellaops-otel-tls + +configMaps: notify-config: data: notify.yaml: | diff --git a/deploy/helm/stellaops/values-prod.yaml b/deploy/helm/stellaops/values-prod.yaml new file mode 100644 index 00000000..03efbad9 --- /dev/null +++ b/deploy/helm/stellaops/values-prod.yaml @@ -0,0 +1,221 @@ +global: + profile: prod + release: + version: "2025.09.2" + channel: stable + manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: stable + stellaops.io/profile: prod + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://stellaops-mongo:27017" + database: "stellaops_notify_prod" + commandTimeoutSeconds: 45 + + authority: + enabled: true + issuer: "https://authority.prod.stella-ops.org" + metadataAddress: "https://authority.prod.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Information +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://authority.prod.stella-ops.org" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + envFrom: + - secretRef: + name: stellaops-prod-core + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "https://licensing.prod.stella-ops.org/introspect" + envFrom: + - secretRef: + name: stellaops-prod-core + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + envFrom: + - secretRef: + name: stellaops-prod-core + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + service: + port: 8445 + env: + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + envFrom: + - secretRef: + name: stellaops-prod-core + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumeClaims: + - name: concelier-jobs + claimName: stellaops-concelier-jobs + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + service: + port: 8444 + env: + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "true" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + envFrom: + - secretRef: + name: stellaops-prod-core + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + replicas: 3 + env: + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + SCANNER__EVENTS__ENABLED: "true" + SCANNER__EVENTS__DRIVER: "redis" + SCANNER__EVENTS__DSN: "" + SCANNER__EVENTS__STREAM: "stella.events" + SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" + SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" + envFrom: + - secretRef: + name: stellaops-prod-core + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Production + envFrom: + - secretRef: + name: stellaops-prod-notify + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + envFrom: + - secretRef: + name: stellaops-prod-core + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + service: + port: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + envFrom: + - secretRef: + name: stellaops-prod-mongo + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: stellaops-mongo-data + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + envFrom: + - secretRef: + name: stellaops-prod-minio + volumeMounts: + - name: minio-data + mountPath: /data + volumeClaims: + - name: minio-data + claimName: stellaops-minio-data + rustfs: + class: infrastructure + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + service: + port: 8080 + command: + - serve + - --listen + - 0.0.0.0:8080 + - --root + - /data + env: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumeMounts: + - name: rustfs-data + mountPath: /data + volumeClaims: + - name: rustfs-data + claimName: stellaops-rustfs-data diff --git a/deploy/helm/stellaops/values-stage.yaml b/deploy/helm/stellaops/values-stage.yaml index 976c256f..bc4dfe17 100644 --- a/deploy/helm/stellaops/values-stage.yaml +++ b/deploy/helm/stellaops/values-stage.yaml @@ -1,13 +1,20 @@ -global: - profile: stage - release: - version: "2025.09.2" +global: + profile: stage + release: + version: "2025.09.2" channel: stable manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" image: pullPolicy: IfNotPresent - labels: - stellaops.io/channel: stable + labels: + stellaops.io/channel: stable + +telemetry: + collector: + enabled: true + defaultTenant: stage + tls: + secretName: stellaops-otel-tls-stage configMaps: notify-config: diff --git a/deploy/helm/stellaops/values.yaml b/deploy/helm/stellaops/values.yaml index 1370a282..83295aa5 100644 --- a/deploy/helm/stellaops/values.yaml +++ b/deploy/helm/stellaops/values.yaml @@ -1,10 +1,37 @@ -global: - release: - version: "" - channel: "" - manifestSha256: "" - profile: "" - image: - pullPolicy: IfNotPresent - labels: {} -services: {} +global: + release: + version: "" + channel: "" + manifestSha256: "" + profile: "" + image: + pullPolicy: IfNotPresent + labels: {} + +telemetry: + collector: + enabled: false + replicas: 1 + image: otel/opentelemetry-collector:0.105.0 + requireClientCert: true + defaultTenant: unknown + logLevel: info + tls: + secretName: "" + certPath: /etc/otel/tls/tls.crt + keyPath: /etc/otel/tls/tls.key + caPath: /etc/otel/tls/ca.crt + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + - key: ca.crt + path: ca.crt + service: + grpcPort: 4317 + httpPort: 4318 + metricsPort: 9464 + resources: {} + +services: {} diff --git a/deploy/telemetry/.gitignore b/deploy/telemetry/.gitignore new file mode 100644 index 00000000..df912870 --- /dev/null +++ b/deploy/telemetry/.gitignore @@ -0,0 +1 @@ +certs/ diff --git a/deploy/telemetry/README.md b/deploy/telemetry/README.md new file mode 100644 index 00000000..926c9a36 --- /dev/null +++ b/deploy/telemetry/README.md @@ -0,0 +1,35 @@ +# Telemetry Collector Assets + +These assets provision the default OpenTelemetry Collector instance required by +`DEVOPS-OBS-50-001`. The collector acts as the secured ingest point for traces, +metrics, and logs emitted by Stella Ops services. + +## Contents + +| File | Purpose | +| ---- | ------- | +| `otel-collector-config.yaml` | Baseline collector configuration (mutual TLS, OTLP receivers, Prometheus exporter). | +| `storage/prometheus.yaml` | Prometheus scrape configuration tuned for the collector and service tenants. | +| `storage/tempo.yaml` | Tempo configuration with multitenancy, WAL, and compaction settings. | +| `storage/loki.yaml` | Loki configuration enabling multitenant log ingestion with retention policies. | +| `storage/tenants/*.yaml` | Per-tenant overrides for Tempo and Loki rate/retention controls. | + +## Development workflow + +1. Generate development certificates (collector + client) using + `ops/devops/telemetry/generate_dev_tls.sh`. +2. Launch the collector via `docker compose -f docker-compose.telemetry.yaml up`. +3. Launch the storage backends (Prometheus, Tempo, Loki) via + `docker compose -f docker-compose.telemetry-storage.yaml up`. +4. Run the smoke test: `python ops/devops/telemetry/smoke_otel_collector.py`. +5. Explore the storage configuration (`storage/README.md`) to tune retention/limits. + +The smoke test sends OTLP traffic over TLS and asserts the collector accepted +traces, metrics, and logs by scraping the Prometheus metrics endpoint. + +## Kubernetes + +The Helm chart consumes the same configuration (see `values.yaml`). Provide TLS +material via a secret referenced by `telemetry.collector.tls.secretName`, +containing `ca.crt`, `tls.crt`, and `tls.key`. Client certificates are required +for ingestion and should be issued by the same CA. diff --git a/deploy/telemetry/otel-collector-config.yaml b/deploy/telemetry/otel-collector-config.yaml new file mode 100644 index 00000000..bc693d4f --- /dev/null +++ b/deploy/telemetry/otel-collector-config.yaml @@ -0,0 +1,67 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + tls: + cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set} + key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set} + client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set} + require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true} + http: + endpoint: 0.0.0.0:4318 + tls: + cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set} + key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set} + client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set} + require_client_certificate: ${STELLAOPS_OTEL_REQUIRE_CLIENT_CERT:true} + +processors: + attributes/tenant-tag: + actions: + - key: tenant.id + action: insert + value: ${STELLAOPS_TENANT_ID:unknown} + batch: + send_batch_size: 1024 + timeout: 5s + +exporters: + logging: + verbosity: normal + prometheus: + endpoint: ${STELLAOPS_OTEL_PROMETHEUS_ENDPOINT:0.0.0.0:9464} + enable_open_metrics: true + metric_expiration: 5m + tls: + cert_file: ${STELLAOPS_OTEL_TLS_CERT:?STELLAOPS_OTEL_TLS_CERT not set} + key_file: ${STELLAOPS_OTEL_TLS_KEY:?STELLAOPS_OTEL_TLS_KEY not set} + client_ca_file: ${STELLAOPS_OTEL_TLS_CA:?STELLAOPS_OTEL_TLS_CA not set} +# Additional OTLP exporters can be configured by extending this section at runtime. +# For example, set STELLAOPS_OTEL_UPSTREAM_ENDPOINT and mount certificates, then +# add the exporter via a sidecar overlay. + +extensions: + health_check: + endpoint: ${STELLAOPS_OTEL_HEALTH_ENDPOINT:0.0.0.0:13133} + pprof: + endpoint: ${STELLAOPS_OTEL_PPROF_ENDPOINT:0.0.0.0:1777} + +service: + telemetry: + logs: + level: ${STELLAOPS_OTEL_LOG_LEVEL:info} + extensions: [health_check, pprof] + pipelines: + traces: + receivers: [otlp] + processors: [attributes/tenant-tag, batch] + exporters: [logging] + metrics: + receivers: [otlp] + processors: [attributes/tenant-tag, batch] + exporters: [logging, prometheus] + logs: + receivers: [otlp] + processors: [attributes/tenant-tag, batch] + exporters: [logging] diff --git a/deploy/telemetry/storage/README.md b/deploy/telemetry/storage/README.md new file mode 100644 index 00000000..b730d5ed --- /dev/null +++ b/deploy/telemetry/storage/README.md @@ -0,0 +1,33 @@ +# Telemetry Storage Stack + +Configuration snippets for the default StellaOps observability backends used in +staging and production environments. The stack comprises: + +- **Prometheus** for metrics (scraping the collector's Prometheus exporter) +- **Tempo** for traces (OTLP ingest via mTLS) +- **Loki** for logs (HTTP ingest with tenant isolation) + +## Files + +| Path | Description | +| ---- | ----------- | +| `prometheus.yaml` | Scrape configuration for the collector (mTLS + bearer token placeholder). | +| `tempo.yaml` | Tempo configuration with multitenancy enabled and local storage paths. | +| `loki.yaml` | Loki configuration enabling per-tenant overrides and boltdb-shipper storage. | +| `tenants/tempo-overrides.yaml` | Example tenant overrides for Tempo (retention, limits). | +| `tenants/loki-overrides.yaml` | Example tenant overrides for Loki (rate limits, retention). | +| `auth/` | Placeholder directory for Prometheus bearer token files (e.g., `token`). | + +These configurations are referenced by the Docker Compose overlay +(`deploy/compose/docker-compose.telemetry-storage.yaml`) and the staging rollout documented in +`docs/ops/telemetry-storage.md`. Adjust paths, credentials, and overrides before running in +connected environments. Place the Prometheus bearer token in `auth/token` when using the +Compose overlay (the directory contains a `.gitkeep` placeholder and is gitignored by default). + +## Security + +- Both Tempo and Loki require mutual TLS. +- Prometheus uses mTLS plus a bearer token that should be minted by Authority. +- Update the overrides files to enforce per-tenant retention/ingestion limits. + +For comprehensive deployment steps see `docs/ops/telemetry-storage.md`. diff --git a/deploy/telemetry/storage/auth/.gitkeep b/deploy/telemetry/storage/auth/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/telemetry/storage/loki.yaml b/deploy/telemetry/storage/loki.yaml new file mode 100644 index 00000000..101b4df3 --- /dev/null +++ b/deploy/telemetry/storage/loki.yaml @@ -0,0 +1,48 @@ +auth_enabled: true + +server: + http_listen_port: 3100 + log_level: info + +common: + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /var/loki + +schema_config: + configs: + - from: 2024-01-01 + store: boltdb-shipper + object_store: filesystem + schema: v13 + index: + prefix: loki_index_ + period: 24h + +storage_config: + filesystem: + directory: /var/loki/chunks + boltdb_shipper: + active_index_directory: /var/loki/index + cache_location: /var/loki/index_cache + shared_store: filesystem + +ruler: + storage: + type: local + local: + directory: /var/loki/rules + rule_path: /tmp/loki-rules + enable_api: true + +limits_config: + enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + max_entries_limit_per_query: 5000 + ingestion_rate_mb: 10 + ingestion_burst_size_mb: 20 + per_tenant_override_config: /etc/telemetry/tenants/loki-overrides.yaml diff --git a/deploy/telemetry/storage/prometheus.yaml b/deploy/telemetry/storage/prometheus.yaml new file mode 100644 index 00000000..e1dcfe4c --- /dev/null +++ b/deploy/telemetry/storage/prometheus.yaml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s + evaluation_interval: 30s + +scrape_configs: + - job_name: "stellaops-otel-collector" + scheme: https + metrics_path: / + tls_config: + ca_file: ${PROMETHEUS_TLS_CA_FILE:-/etc/telemetry/tls/ca.crt} + cert_file: ${PROMETHEUS_TLS_CERT_FILE:-/etc/telemetry/tls/client.crt} + key_file: ${PROMETHEUS_TLS_KEY_FILE:-/etc/telemetry/tls/client.key} + insecure_skip_verify: false + authorization: + type: Bearer + credentials_file: ${PROMETHEUS_BEARER_TOKEN_FILE:-/etc/telemetry/auth/token} + static_configs: + - targets: + - ${PROMETHEUS_COLLECTOR_TARGET:-stellaops-otel-collector:9464} diff --git a/deploy/telemetry/storage/tempo.yaml b/deploy/telemetry/storage/tempo.yaml new file mode 100644 index 00000000..976e517b --- /dev/null +++ b/deploy/telemetry/storage/tempo.yaml @@ -0,0 +1,56 @@ +multitenancy_enabled: true +usage_report: + reporting_enabled: false + +server: + http_listen_port: 3200 + log_level: info + +distributor: + receivers: + otlp: + protocols: + grpc: + tls: + cert_file: ${TEMPO_TLS_CERT_FILE:-/etc/telemetry/tls/server.crt} + key_file: ${TEMPO_TLS_KEY_FILE:-/etc/telemetry/tls/server.key} + client_ca_file: ${TEMPO_TLS_CA_FILE:-/etc/telemetry/tls/ca.crt} + require_client_cert: true + http: + tls: + cert_file: ${TEMPO_TLS_CERT_FILE:-/etc/telemetry/tls/server.crt} + key_file: ${TEMPO_TLS_KEY_FILE:-/etc/telemetry/tls/server.key} + client_ca_file: ${TEMPO_TLS_CA_FILE:-/etc/telemetry/tls/ca.crt} + require_client_cert: true + +ingester: + lifecycler: + ring: + instance_availability_zone: ${TEMPO_ZONE:-zone-a} + trace_idle_period: 10s + max_block_bytes: 1_048_576 + +compactor: + compaction: + block_retention: 168h + +metrics_generator: + registry: + external_labels: + cluster: stellaops + +storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + metrics: + backend: prometheus + +overrides: + defaults: + ingestion_rate_limit_bytes: 1048576 + max_traces_per_user: 200000 + per_tenant_override_config: /etc/telemetry/tenants/tempo-overrides.yaml diff --git a/deploy/telemetry/storage/tenants/loki-overrides.yaml b/deploy/telemetry/storage/tenants/loki-overrides.yaml new file mode 100644 index 00000000..b0680f31 --- /dev/null +++ b/deploy/telemetry/storage/tenants/loki-overrides.yaml @@ -0,0 +1,19 @@ +# Example Loki per-tenant overrides +# Adjust according to https://grafana.com/docs/loki/latest/configuration/#limits_config + +stellaops-dev: + ingestion_rate_mb: 10 + ingestion_burst_size_mb: 20 + max_global_streams_per_user: 5000 + retention_period: 168h + +stellaops-stage: + ingestion_rate_mb: 20 + ingestion_burst_size_mb: 40 + max_global_streams_per_user: 10000 + retention_period: 336h + +__default__: + ingestion_rate_mb: 5 + ingestion_burst_size_mb: 10 + retention_period: 72h diff --git a/deploy/telemetry/storage/tenants/tempo-overrides.yaml b/deploy/telemetry/storage/tenants/tempo-overrides.yaml new file mode 100644 index 00000000..26066897 --- /dev/null +++ b/deploy/telemetry/storage/tenants/tempo-overrides.yaml @@ -0,0 +1,16 @@ +# Example Tempo per-tenant overrides +# Consult https://grafana.com/docs/tempo/latest/configuration/#limits-configuration +# before applying in production. + +stellaops-dev: + traces_per_second_limit: 100000 + max_bytes_per_trace: 10485760 + max_search_bytes_per_trace: 20971520 + +stellaops-stage: + traces_per_second_limit: 200000 + max_bytes_per_trace: 20971520 + +__default__: + traces_per_second_limit: 50000 + max_bytes_per_trace: 5242880 diff --git a/deploy/tools/check-channel-alignment.py b/deploy/tools/check-channel-alignment.py new file mode 100644 index 00000000..d92dd0e1 --- /dev/null +++ b/deploy/tools/check-channel-alignment.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +""" +Ensure deployment bundles reference the images defined in a release manifest. + +Usage: + ./deploy/tools/check-channel-alignment.py \ + --release deploy/releases/2025.10-edge.yaml \ + --target deploy/helm/stellaops/values-dev.yaml \ + --target deploy/compose/docker-compose.dev.yaml + +For every target file, the script scans `image:` declarations and verifies that +any image belonging to a repository listed in the release manifest matches the +exact digest or tag recorded there. Images outside of the manifest (for example, +supporting services such as `nats`) are ignored. +""" + +from __future__ import annotations + +import argparse +import pathlib +import re +import sys +from typing import Dict, Iterable, List, Optional, Set + +IMAGE_LINE = re.compile(r"^\s*image:\s*['\"]?(?P\S+)['\"]?\s*$") + + +def extract_images(path: pathlib.Path) -> List[str]: + images: List[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + match = IMAGE_LINE.match(line) + if match: + images.append(match.group("image")) + return images + + +def image_repo(image: str) -> str: + if "@" in image: + return image.split("@", 1)[0] + # Split on the last colon to preserve registries with ports (e.g. localhost:5000) + if ":" in image: + prefix, tag = image.rsplit(":", 1) + if "/" in tag: + # handle digestive colon inside path (unlikely) + return image + return prefix + return image + + +def load_release_map(release_path: pathlib.Path) -> Dict[str, str]: + release_map: Dict[str, str] = {} + for image in extract_images(release_path): + repo = image_repo(image) + release_map[repo] = image + return release_map + + +def check_target( + target_path: pathlib.Path, + release_map: Dict[str, str], + ignore_repos: Set[str], +) -> List[str]: + errors: List[str] = [] + for image in extract_images(target_path): + repo = image_repo(image) + if repo in ignore_repos: + continue + if repo not in release_map: + continue + expected = release_map[repo] + if image != expected: + errors.append( + f"{target_path}: {image} does not match release value {expected}" + ) + return errors + + +def parse_args(argv: Optional[Iterable[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--release", + required=True, + type=pathlib.Path, + help="Path to the release manifest (YAML)", + ) + parser.add_argument( + "--target", + action="append", + required=True, + type=pathlib.Path, + help="Deployment profile to validate against the release manifest", + ) + parser.add_argument( + "--ignore-repo", + action="append", + default=[], + help="Repository prefix to ignore (may be repeated)", + ) + return parser.parse_args(argv) + + +def main(argv: Optional[Iterable[str]] = None) -> int: + args = parse_args(argv) + + release_map = load_release_map(args.release) + ignore_repos = {repo.rstrip("/") for repo in args.ignore_repo} + + if not release_map: + print(f"error: no images found in release manifest {args.release}", file=sys.stderr) + return 2 + + total_errors: List[str] = [] + for target in args.target: + if not target.exists(): + total_errors.append(f"{target}: file not found") + continue + total_errors.extend(check_target(target, release_map, ignore_repos)) + + if total_errors: + print("✖ channel alignment check failed:", file=sys.stderr) + for err in total_errors: + print(f" - {err}", file=sys.stderr) + return 1 + + print("✓ deployment profiles reference release images for the inspected repositories.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/tools/validate-profiles.sh b/deploy/tools/validate-profiles.sh index f8ac4af1..5680f0f5 100644 --- a/deploy/tools/validate-profiles.sh +++ b/deploy/tools/validate-profiles.sh @@ -1,53 +1,61 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -COMPOSE_DIR="$ROOT_DIR/compose" -HELM_DIR="$ROOT_DIR/helm/stellaops" - -compose_profiles=( - "docker-compose.dev.yaml:env/dev.env.example" - "docker-compose.stage.yaml:env/stage.env.example" - "docker-compose.airgap.yaml:env/airgap.env.example" - "docker-compose.mirror.yaml:env/mirror.env.example" -) - -docker_ready=false -if command -v docker >/dev/null 2>&1; then - if docker compose version >/dev/null 2>&1; then - docker_ready=true - else - echo "⚠️ docker CLI present but Compose plugin unavailable; skipping compose validation" >&2 - fi -else - echo "⚠️ docker CLI not found; skipping compose validation" >&2 -fi - -if [[ "$docker_ready" == "true" ]]; then - for entry in "${compose_profiles[@]}"; do - IFS=":" read -r compose_file env_file <<<"$entry" - printf '→ validating %s with %s\n' "$compose_file" "$env_file" - docker compose \ - --env-file "$COMPOSE_DIR/$env_file" \ - -f "$COMPOSE_DIR/$compose_file" config >/dev/null - done -fi - -helm_values=( - "$HELM_DIR/values-dev.yaml" - "$HELM_DIR/values-stage.yaml" - "$HELM_DIR/values-airgap.yaml" - "$HELM_DIR/values-mirror.yaml" -) - -if command -v helm >/dev/null 2>&1; then - for values in "${helm_values[@]}"; do - printf '→ linting Helm chart with %s\n' "$(basename "$values")" - helm lint "$HELM_DIR" -f "$values" - helm template test-release "$HELM_DIR" -f "$values" >/dev/null - done -else - echo "⚠️ helm CLI not found; skipping Helm lint/template" >&2 -fi - -printf 'Profiles validated (where tooling was available).\n' +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_DIR="$ROOT_DIR/compose" +HELM_DIR="$ROOT_DIR/helm/stellaops" + +compose_profiles=( + "docker-compose.dev.yaml:env/dev.env.example" + "docker-compose.stage.yaml:env/stage.env.example" + "docker-compose.prod.yaml:env/prod.env.example" + "docker-compose.airgap.yaml:env/airgap.env.example" + "docker-compose.mirror.yaml:env/mirror.env.example" + "docker-compose.telemetry.yaml:" + "docker-compose.telemetry-storage.yaml:" +) + +docker_ready=false +if command -v docker >/dev/null 2>&1; then + if docker compose version >/dev/null 2>&1; then + docker_ready=true + else + echo "⚠️ docker CLI present but Compose plugin unavailable; skipping compose validation" >&2 + fi +else + echo "⚠️ docker CLI not found; skipping compose validation" >&2 +fi + +if [[ "$docker_ready" == "true" ]]; then + for entry in "${compose_profiles[@]}"; do + IFS=":" read -r compose_file env_file <<<"$entry" + printf '→ validating %s with %s\n' "$compose_file" "$env_file" + if [[ -n "$env_file" ]]; then + docker compose \ + --env-file "$COMPOSE_DIR/$env_file" \ + -f "$COMPOSE_DIR/$compose_file" config >/dev/null + else + docker compose -f "$COMPOSE_DIR/$compose_file" config >/dev/null + fi + done +fi + +helm_values=( + "$HELM_DIR/values-dev.yaml" + "$HELM_DIR/values-stage.yaml" + "$HELM_DIR/values-prod.yaml" + "$HELM_DIR/values-airgap.yaml" + "$HELM_DIR/values-mirror.yaml" +) + +if command -v helm >/dev/null 2>&1; then + for values in "${helm_values[@]}"; do + printf '→ linting Helm chart with %s\n' "$(basename "$values")" + helm lint "$HELM_DIR" -f "$values" + helm template test-release "$HELM_DIR" -f "$values" >/dev/null + done +else + echo "⚠️ helm CLI not found; skipping Helm lint/template" >&2 +fi + +printf 'Profiles validated (where tooling was available).\n' diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index d08809db..df84da3d 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -629,6 +629,32 @@ See `docs/dev/32_AUTH_CLIENT_GUIDE.md` for recommended profiles (online vs. air- | `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | | `stellaops-cli runtime policy test` | Ask Scanner.WebService for runtime verdicts (Webhook parity) | `--image/-i ` (repeatable, comma/space lists supported)
`--file/-f `
`--namespace/--ns `
`--label/-l key=value` (repeatable)
`--json` | Posts to `POST /api/v1/scanner/policy/runtime`, deduplicates image digests, and prints TTL/policy revision plus per-image columns for signed state, SBOM referrers, quieted-by metadata, confidence, Rekor attestation (uuid + verified flag), and recently observed build IDs (shortened for readability). Accepts newline/whitespace-delimited stdin when piped; `--json` emits the raw response without additional logging. | +#### Example: Pivot from runtime verdicts to debug symbols + +```bash +$ stellaops-cli runtime policy test \ + --image ghcr.io/acme/payments@sha256:4f7d55f6... \ + --namespace payments + +Image Digest Signed SBOM Build IDs TTL +ghcr.io/acme/payments@sha256:4f7d55f6... yes present 5f0c7c3c..., 1122aabbccddeeff... 04:59:55 +``` + +1. Copy one of the hashes (e.g. `5f0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789`) and locate the bundled debug artefact: + ```bash + ls offline-kit/debug/.build-id/5f/0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789.debug + ``` +2. Confirm the running binary advertises the same GNU build-id: + ```bash + readelf -n /proc/$(pgrep -f payments-api | head -n1)/exe | grep -i 'Build ID' + ``` +3. If you operate a debuginfod mirror backed by the Offline Kit tree, resolve symbols with: + ```bash + debuginfod-find debuginfo 5f0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789 >/tmp/payments-api.debug + ``` + +See [Offline Kit step 0](24_OFFLINE_KIT.md#0-prepare-the-debug-store) for instructions on mirroring the debug store before packaging. + `POST /api/v1/scanner/policy/runtime` responds with one entry per digest. Each result now includes: - `policyVerdict` (`pass|warn|fail|error`), `signed`, and `hasSbomReferrers` parity with the webhook contract. @@ -739,7 +765,7 @@ For offline workflows, configure `StellaOps:Offline:KitsDirectory` (or `STELLAOP "ClientSecret": "REDACTED", "Username": "", "Password": "", - "Scope": "concelier.jobs.trigger", + "Scope": "concelier.jobs.trigger advisory:ingest advisory:read", "TokenCacheDirectory": "" } } diff --git a/docs/10_CONCELIER_CLI_QUICKSTART.md b/docs/10_CONCELIER_CLI_QUICKSTART.md index 24c212fe..3f352fda 100644 --- a/docs/10_CONCELIER_CLI_QUICKSTART.md +++ b/docs/10_CONCELIER_CLI_QUICKSTART.md @@ -1,61 +1,61 @@ -# 10 · Concelier + CLI Quickstart - -This guide walks through configuring the Concelier web service and the `stellaops-cli` -tool so an operator can ingest advisories, merge them, and publish exports from a -single workstation. It focuses on deployment-facing surfaces only (configuration, -runtime wiring, CLI usage) and leaves connector/internal customization for later. - ---- - -## 0 · Prerequisites - -- .NET SDK **10.0.100-preview** (matches `global.json`) -- MongoDB instance reachable from the host (local Docker or managed) -- `trivy-db` binary on `PATH` for Trivy exports (and `oras` if publishing to OCI) -- Plugin assemblies present in `StellaOps.Concelier.PluginBinaries/` (already included in the repo) -- Optional: Docker/Podman runtime if you plan to run scanners locally - -> **Tip** – air-gapped installs should preload `trivy-db` and `oras` binaries into the -> runner image since Concelier never fetches them dynamically. - ---- - -## 1 · Configure Concelier - -1. Copy the sample config to the expected location (CI/CD pipelines can stamp values - into this file during deployment—see the “Deployment automation” note below): - - ```bash - mkdir -p etc - cp etc/concelier.yaml.sample etc/concelier.yaml - ``` - -2. Edit `etc/concelier.yaml` and update the MongoDB DSN (and optional database name). - The default template configures plug-in discovery to look in `StellaOps.Concelier.PluginBinaries/` - and disables remote telemetry exporters by default. - -3. (Optional) Override settings via environment variables. All keys are prefixed with - `CONCELIER_`. Example: - - ```bash - export CONCELIER_STORAGE__DSN="mongodb://user:pass@mongo:27017/concelier" - export CONCELIER_TELEMETRY__ENABLETRACING=false - ``` - -4. Start the web service from the repository root: - - ```bash - dotnet run --project src/StellaOps.Concelier.WebService - ``` - - On startup Concelier validates the options, boots MongoDB indexes, loads plug-ins, - and exposes: - - - `GET /health` – returns service status and telemetry settings - - `GET /ready` – performs a MongoDB `ping` - - `GET /jobs` + `POST /jobs/{kind}` – inspect and trigger connector/export jobs - - > **Security note** – authentication now ships via StellaOps Authority. Keep +# 10 · Concelier + CLI Quickstart + +This guide walks through configuring the Concelier web service and the `stellaops-cli` +tool so an operator can ingest advisories, merge them, and publish exports from a +single workstation. It focuses on deployment-facing surfaces only (configuration, +runtime wiring, CLI usage) and leaves connector/internal customization for later. + +--- + +## 0 · Prerequisites + +- .NET SDK **10.0.100-preview** (matches `global.json`) +- MongoDB instance reachable from the host (local Docker or managed) +- `trivy-db` binary on `PATH` for Trivy exports (and `oras` if publishing to OCI) +- Plugin assemblies present in `StellaOps.Concelier.PluginBinaries/` (already included in the repo) +- Optional: Docker/Podman runtime if you plan to run scanners locally + +> **Tip** – air-gapped installs should preload `trivy-db` and `oras` binaries into the +> runner image since Concelier never fetches them dynamically. + +--- + +## 1 · Configure Concelier + +1. Copy the sample config to the expected location (CI/CD pipelines can stamp values + into this file during deployment—see the “Deployment automation” note below): + + ```bash + mkdir -p etc + cp etc/concelier.yaml.sample etc/concelier.yaml + ``` + +2. Edit `etc/concelier.yaml` and update the MongoDB DSN (and optional database name). + The default template configures plug-in discovery to look in `StellaOps.Concelier.PluginBinaries/` + and disables remote telemetry exporters by default. + +3. (Optional) Override settings via environment variables. All keys are prefixed with + `CONCELIER_`. Example: + + ```bash + export CONCELIER_STORAGE__DSN="mongodb://user:pass@mongo:27017/concelier" + export CONCELIER_TELEMETRY__ENABLETRACING=false + ``` + +4. Start the web service from the repository root: + + ```bash + dotnet run --project src/StellaOps.Concelier.WebService + ``` + + On startup Concelier validates the options, boots MongoDB indexes, loads plug-ins, + and exposes: + + - `GET /health` – returns service status and telemetry settings + - `GET /ready` – performs a MongoDB `ping` + - `GET /jobs` + `POST /jobs/{kind}` – inspect and trigger connector/export jobs + + > **Security note** – authentication now ships via StellaOps Authority. Keep > `authority.allowAnonymousFallback: true` only during the staged rollout and > disable it before **2025-12-31 UTC** so tokens become mandatory. @@ -66,230 +66,244 @@ Rollout checkpoints for the two Authority toggles: | **Validation (staging)** | `true` | `true` | Verify token issuance, CLI scopes, and audit log noise without breaking cron jobs. | Watch `Concelier.Authorization.Audit` for `bypass=True` events and scope gaps; confirm CLI `auth status` succeeds. | | **Cutover rehearsal** | `true` | `false` | Exercise production-style enforcement before the deadline; ensure only approved maintenance ranges remain in `bypassNetworks`. | Expect some HTTP 401s; verify `web.jobs.triggered` metrics flatten for unauthenticated calls and audit logs highlight missing tokens. | | **Enforced (steady state)** | `true` | `false` | Production baseline after the 2025-12-31 UTC cutoff. | Alert on new `bypass=True` entries and on repeated 401 bursts; correlate with Authority availability dashboards. | - -### Authority companion configuration (preview) - -1. Copy the Authority sample configuration: - - ```bash - cp etc/authority.yaml.sample etc/authority.yaml - ``` - -2. Update the issuer URL, token lifetimes, and plug-in descriptors to match your - environment. Authority expects per-plugin manifests in `etc/authority.plugins/`; - sample `standard.yaml` and `ldap.yaml` files are provided as starting points. - For air-gapped installs keep the default plug-in binary directory - (`../StellaOps.Authority.PluginBinaries`) so packaged plug-ins load without outbound access. - -3. Environment variables prefixed with `STELLAOPS_AUTHORITY_` override individual - fields. Example: - - ```bash - export STELLAOPS_AUTHORITY__ISSUER="https://authority.stella-ops.local" - export STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0="/srv/authority/plugins" - ``` - ---- - -## 2 · Configure the CLI - -The CLI reads configuration from JSON/YAML files *and* environment variables. The -defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at runtime. - -| Setting | Environment variable | Default | Purpose | -| ------- | -------------------- | ------- | ------- | -| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Concelier web service | -| `ApiKey` | `API_KEY` | _empty_ | Reserved for legacy key auth; leave empty when using Authority | -| `ScannerCacheDirectory` | `STELLAOPS_SCANNER_CACHE_DIRECTORY` | `scanners` | Local cache folder | -| `ResultsDirectory` | `STELLAOPS_RESULTS_DIRECTORY` | `results` | Where scan outputs are written | -| `Authority.Url` | `STELLAOPS_AUTHORITY_URL` | _empty_ | StellaOps Authority issuer/token endpoint | -| `Authority.ClientId` | `STELLAOPS_AUTHORITY_CLIENT_ID` | _empty_ | Client identifier for the CLI | -| `Authority.ClientSecret` | `STELLAOPS_AUTHORITY_CLIENT_SECRET` | _empty_ | Client secret (omit when using username/password grant) | -| `Authority.Username` | `STELLAOPS_AUTHORITY_USERNAME` | _empty_ | Username for password grant flows | -| `Authority.Password` | `STELLAOPS_AUTHORITY_PASSWORD` | _empty_ | Password for password grant flows | -| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `concelier.jobs.trigger` | OAuth scope requested for backend operations | -| `Authority.TokenCacheDirectory` | `STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR` | `~/.stellaops/tokens` | Directory that persists cached tokens | -| `Authority.Resilience.EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly retry handler for Authority HTTP calls | -| `Authority.Resilience.RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma- or space-separated backoff delays (hh:mm:ss) | -| `Authority.Resilience.AllowOfflineCacheFallback` | `STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK` | `true` | Allow CLI to reuse cached discovery/JWKS metadata when Authority is offline | -| `Authority.Resilience.OfflineCacheTolerance` | `STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE` | `00:10:00` | Additional tolerance window applied to cached metadata | - -Example bootstrap: - -```bash -export STELLAOPS_BACKEND_URL="http://localhost:5000" -export STELLAOPS_RESULTS_DIRECTORY="$HOME/.stellaops/results" -export STELLAOPS_AUTHORITY_URL="https://authority.local" -export STELLAOPS_AUTHORITY_CLIENT_ID="concelier-cli" -export STELLAOPS_AUTHORITY_CLIENT_SECRET="s3cr3t" -dotnet run --project src/StellaOps.Cli -- db merge - -# Acquire a bearer token and confirm cache state -dotnet run --project src/StellaOps.Cli -- auth login -dotnet run --project src/StellaOps.Cli -- auth status -dotnet run --project src/StellaOps.Cli -- auth whoami -``` - -Refer to `docs/dev/32_AUTH_CLIENT_GUIDE.md` for deeper guidance on tuning retry/offline settings and rollout checklists. - -To persist configuration, you can create `stellaops-cli.yaml` next to the binary or -rely on environment variables for ephemeral runners. - ---- - -## 3 · Operating Workflow - -1. **Trigger connector fetch stages** - - ```bash - dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage fetch - dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage parse - dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage map - ``` - - Use `--mode resume` when continuing from a previous window: - - ```bash - dotnet run --project src/StellaOps.Cli -- db fetch --source redhat --stage fetch --mode resume - ``` - -2. **Merge canonical advisories** - - ```bash - dotnet run --project src/StellaOps.Cli -- db merge - ``` - -3. **Produce exports** - - ```bash - # JSON tree (vuln-list style) - dotnet run --project src/StellaOps.Cli -- db export --format json - - # Trivy DB (delta example) - dotnet run --project src/StellaOps.Cli -- db export --format trivy-db --delta - ``` - - Concelier always produces a deterministic OCI layout. The first run after a clean - bootstrap emits a **full** baseline; subsequent `--delta` runs reuse the previous - baseline’s blobs when only JSON manifests change. If the exporter detects that a - prior delta is still active (i.e., `LastDeltaDigest` is recorded) it automatically - upgrades the next run to a full export and resets the baseline so operators never - chain deltas indefinitely. The CLI exposes `--publish-full/--publish-delta` (for - ORAS pushes) and `--include-full/--include-delta` (for offline bundles) should you - need to override the defaults interactively. - - **Smoke-check delta reuse:** after the first baseline completes, run the export a - second time with `--delta` and verify that the new directory reports `mode=delta` - while reusing the previous layer blob. - - ```bash - export_root=${CONCELIER_EXPORT_ROOT:-exports/trivy} - base=$(ls -1d "$export_root"/* | sort | tail -n2 | head -n1) - delta=$(ls -1d "$export_root"/* | sort | tail -n1) - - jq -r '.mode,.baseExportId' "$delta/metadata.json" - - base_manifest=$(jq -r '.manifests[0].digest' "$base/index.json") - delta_manifest=$(jq -r '.manifests[0].digest' "$delta/index.json") - printf 'baseline manifest: %s\ndelta manifest: %s\n' "$base_manifest" "$delta_manifest" - - layer_digest=$(jq -r '.layers[0].digest' "$base/blobs/sha256/${base_manifest#sha256:}") - cmp "$base/blobs/sha256/${layer_digest#sha256:}" \ - "$delta/blobs/sha256/${layer_digest#sha256:}" - ``` - - `cmp` returning exit code `0` confirms the delta export reuses the baseline’s - `db.tar.gz` layer instead of rebuilding it. - -4. **Manage scanners (optional)** - - ```bash - dotnet run --project src/StellaOps.Cli -- scanner download --channel stable - dotnet run --project src/StellaOps.Cli -- scan run --entry scanners/latest/Scanner.dll --target ./sboms - dotnet run --project src/StellaOps.Cli -- scan upload --file results/scan-001.json - ``` - -Add `--verbose` to any command for structured console logs. All commands honour -`Ctrl+C` cancellation and exit with non-zero status codes when the backend returns -a problem document. - ---- - -## 4 · Verification Checklist - -- Concelier `/health` returns `"status":"healthy"` and Storage bootstrap is marked - complete after startup. -- CLI commands return HTTP 202 with a `Location` header (job tracking URL) when - triggering Concelier jobs. -- Export artefacts are materialised under the configured output directories and - their manifests record digests. -- MongoDB contains the expected `document`, `dto`, `advisory`, and `export_state` - collections after a run. - ---- - -## 5 · Deployment Automation - -- Treat `etc/concelier.yaml.sample` as the canonical template. CI/CD should copy it to - the deployment artifact and replace placeholders (DSN, telemetry endpoints, cron - overrides) with environment-specific secrets. -- Keep secret material (Mongo credentials, OTLP tokens) outside of the repository; - inject them via secret stores or pipeline variables at stamp time. -- When building container images, include `trivy-db` (and `oras` if used) so air-gapped - clusters do not need outbound downloads at runtime. - ---- - -## 5 · Next Steps - -- Enable authority-backed authentication in non-production first. Set - `authority.enabled: true` while keeping `authority.allowAnonymousFallback: true` - to observe logs, then flip it to `false` before 2025-12-31 UTC to enforce tokens. -- Automate the workflow above via CI/CD (compose stack or Kubernetes CronJobs). -- Pair with the Concelier connector teams when enabling additional sources so their - module-specific requirements are pulled in safely. - ---- - + +### Authority companion configuration (preview) + +1. Copy the Authority sample configuration: + + ```bash + cp etc/authority.yaml.sample etc/authority.yaml + ``` + +2. Update the issuer URL, token lifetimes, and plug-in descriptors to match your + environment. Authority expects per-plugin manifests in `etc/authority.plugins/`; + sample `standard.yaml` and `ldap.yaml` files are provided as starting points. + For air-gapped installs keep the default plug-in binary directory + (`../StellaOps.Authority.PluginBinaries`) so packaged plug-ins load without outbound access. + +3. Environment variables prefixed with `STELLAOPS_AUTHORITY_` override individual + fields. Example: + + ```bash + export STELLAOPS_AUTHORITY__ISSUER="https://authority.stella-ops.local" + export STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0="/srv/authority/plugins" + ``` + +--- + +## 2 · Configure the CLI + +The CLI reads configuration from JSON/YAML files *and* environment variables. The +defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at runtime. + +| Setting | Environment variable | Default | Purpose | +| ------- | -------------------- | ------- | ------- | +| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Concelier web service | +| `ApiKey` | `API_KEY` | _empty_ | Reserved for legacy key auth; leave empty when using Authority | +| `ScannerCacheDirectory` | `STELLAOPS_SCANNER_CACHE_DIRECTORY` | `scanners` | Local cache folder | +| `ResultsDirectory` | `STELLAOPS_RESULTS_DIRECTORY` | `results` | Where scan outputs are written | +| `Authority.Url` | `STELLAOPS_AUTHORITY_URL` | _empty_ | StellaOps Authority issuer/token endpoint | +| `Authority.ClientId` | `STELLAOPS_AUTHORITY_CLIENT_ID` | _empty_ | Client identifier for the CLI | +| `Authority.ClientSecret` | `STELLAOPS_AUTHORITY_CLIENT_SECRET` | _empty_ | Client secret (omit when using username/password grant) | +| `Authority.Username` | `STELLAOPS_AUTHORITY_USERNAME` | _empty_ | Username for password grant flows | +| `Authority.Password` | `STELLAOPS_AUTHORITY_PASSWORD` | _empty_ | Password for password grant flows | +| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `concelier.jobs.trigger advisory:ingest` | Space-separated OAuth scopes requested for backend operations | +| `Authority.TokenCacheDirectory` | `STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR` | `~/.stellaops/tokens` | Directory that persists cached tokens | +| `Authority.Resilience.EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly retry handler for Authority HTTP calls | +| `Authority.Resilience.RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma- or space-separated backoff delays (hh:mm:ss) | +| `Authority.Resilience.AllowOfflineCacheFallback` | `STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK` | `true` | Allow CLI to reuse cached discovery/JWKS metadata when Authority is offline | +| `Authority.Resilience.OfflineCacheTolerance` | `STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE` | `00:10:00` | Additional tolerance window applied to cached metadata | + +Example bootstrap: + +```bash +export STELLAOPS_BACKEND_URL="http://localhost:5000" +export STELLAOPS_RESULTS_DIRECTORY="$HOME/.stellaops/results" +export STELLAOPS_AUTHORITY_URL="https://authority.local" +export STELLAOPS_AUTHORITY_CLIENT_ID="concelier-cli" +export STELLAOPS_AUTHORITY_CLIENT_SECRET="s3cr3t" +export STELLAOPS_AUTHORITY_SCOPE="concelier.jobs.trigger advisory:ingest advisory:read" +dotnet run --project src/StellaOps.Cli -- db merge + +# Acquire a bearer token and confirm cache state +dotnet run --project src/StellaOps.Cli -- auth login +dotnet run --project src/StellaOps.Cli -- auth status +dotnet run --project src/StellaOps.Cli -- auth whoami +``` + +Refer to `docs/dev/32_AUTH_CLIENT_GUIDE.md` for deeper guidance on tuning retry/offline settings and rollout checklists. + +To persist configuration, you can create `stellaops-cli.yaml` next to the binary or +rely on environment variables for ephemeral runners. + +--- + +## 3 · Operating Workflow + +1. **Trigger connector fetch stages** + + ```bash + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage fetch + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage parse + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage map + ``` + + Use `--mode resume` when continuing from a previous window: + + ```bash + dotnet run --project src/StellaOps.Cli -- db fetch --source redhat --stage fetch --mode resume + ``` + +2. **Merge canonical advisories** + + ```bash + dotnet run --project src/StellaOps.Cli -- db merge + ``` + +3. **Produce exports** + + ```bash + # JSON tree (vuln-list style) + dotnet run --project src/StellaOps.Cli -- db export --format json + + # Trivy DB (delta example) + dotnet run --project src/StellaOps.Cli -- db export --format trivy-db --delta + ``` + + Concelier always produces a deterministic OCI layout. The first run after a clean + bootstrap emits a **full** baseline; subsequent `--delta` runs reuse the previous + baseline’s blobs when only JSON manifests change. If the exporter detects that a + prior delta is still active (i.e., `LastDeltaDigest` is recorded) it automatically + upgrades the next run to a full export and resets the baseline so operators never + chain deltas indefinitely. The CLI exposes `--publish-full/--publish-delta` (for + ORAS pushes) and `--include-full/--include-delta` (for offline bundles) should you + need to override the defaults interactively. + + **Smoke-check delta reuse:** after the first baseline completes, run the export a + second time with `--delta` and verify that the new directory reports `mode=delta` + while reusing the previous layer blob. + + ```bash + export_root=${CONCELIER_EXPORT_ROOT:-exports/trivy} + base=$(ls -1d "$export_root"/* | sort | tail -n2 | head -n1) + delta=$(ls -1d "$export_root"/* | sort | tail -n1) + + jq -r '.mode,.baseExportId' "$delta/metadata.json" + + base_manifest=$(jq -r '.manifests[0].digest' "$base/index.json") + delta_manifest=$(jq -r '.manifests[0].digest' "$delta/index.json") + printf 'baseline manifest: %s\ndelta manifest: %s\n' "$base_manifest" "$delta_manifest" + + layer_digest=$(jq -r '.layers[0].digest' "$base/blobs/sha256/${base_manifest#sha256:}") + cmp "$base/blobs/sha256/${layer_digest#sha256:}" \ + "$delta/blobs/sha256/${layer_digest#sha256:}" + ``` + + `cmp` returning exit code `0` confirms the delta export reuses the baseline’s + `db.tar.gz` layer instead of rebuilding it. + +4. **Manage scanners (optional)** + + ```bash + dotnet run --project src/StellaOps.Cli -- scanner download --channel stable + dotnet run --project src/StellaOps.Cli -- scan run --entry scanners/latest/Scanner.dll --target ./sboms + dotnet run --project src/StellaOps.Cli -- scan upload --file results/scan-001.json + ``` + +Add `--verbose` to any command for structured console logs. All commands honour +`Ctrl+C` cancellation and exit with non-zero status codes when the backend returns +a problem document. + +--- + +## 4 · Verification Checklist + +- Concelier `/health` returns `"status":"healthy"` and Storage bootstrap is marked + complete after startup. +- CLI commands return HTTP 202 with a `Location` header (job tracking URL) when + triggering Concelier jobs. +- Export artefacts are materialised under the configured output directories and + their manifests record digests. +- MongoDB contains the expected `document`, `dto`, `advisory`, and `export_state` + collections after a run. + +--- + +## 5 · Deployment Automation + +- Treat `etc/concelier.yaml.sample` as the canonical template. CI/CD should copy it to + the deployment artifact and replace placeholders (DSN, telemetry endpoints, cron + overrides) with environment-specific secrets. +- Keep secret material (Mongo credentials, OTLP tokens) outside of the repository; + inject them via secret stores or pipeline variables at stamp time. +- When building container images, include `trivy-db` (and `oras` if used) so air-gapped + clusters do not need outbound downloads at runtime. + +--- + +## 5 · Next Steps + +- Enable authority-backed authentication in non-production first. Set + `authority.enabled: true` while keeping `authority.allowAnonymousFallback: true` + to observe logs, then flip it to `false` before 2025-12-31 UTC to enforce tokens. +- Automate the workflow above via CI/CD (compose stack or Kubernetes CronJobs). +- Pair with the Concelier connector teams when enabling additional sources so their + module-specific requirements are pulled in safely. + +--- + ## 6 · Authority Integration - Concelier now authenticates callers through StellaOps Authority using OAuth 2.0 resource server flows. Populate the `authority` block in `concelier.yaml`: - - ```yaml - authority: - enabled: true - allowAnonymousFallback: false # keep true only during the staged rollout window - issuer: "https://authority.example.org" - audiences: - - "api://concelier" - requiredScopes: - - "concelier.jobs.trigger" - clientId: "concelier-jobs" - clientSecretFile: "../secrets/concelier-jobs.secret" - clientScopes: - - "concelier.jobs.trigger" - bypassNetworks: - - "127.0.0.1/32" - - "::1/128" - ``` - -- Store the client secret outside of source control. Either provide it via - `authority.clientSecret` (environment variable `CONCELIER_AUTHORITY__CLIENTSECRET`) - or point `authority.clientSecretFile` to a file mounted at runtime. -- Cron jobs running on the same host can keep using the API thanks to the loopback - bypass mask. Add additional CIDR ranges as needed; every bypass is logged. -- Export the same configuration to Kubernetes or systemd by setting environment - variables such as: - - ```bash - export CONCELIER_AUTHORITY__ENABLED=true - export CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false - export CONCELIER_AUTHORITY__ISSUER="https://authority.example.org" - export CONCELIER_AUTHORITY__CLIENTID="concelier-jobs" - export CONCELIER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/concelier/authority-client" - ``` - + + ```yaml + authority: + enabled: true + allowAnonymousFallback: false # keep true only during the staged rollout window + issuer: "https://authority.example.org" + audiences: + - "api://concelier" + requiredScopes: + - "concelier.jobs.trigger" + - "advisory:read" + - "advisory:ingest" + requiredTenants: + - "tenant-default" + clientId: "concelier-jobs" + clientSecretFile: "../secrets/concelier-jobs.secret" + clientScopes: + - "concelier.jobs.trigger" + - "advisory:read" + - "advisory:ingest" + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + ``` + +- Store the client secret outside of source control. Either provide it via + `authority.clientSecret` (environment variable `CONCELIER_AUTHORITY__CLIENTSECRET`) + or point `authority.clientSecretFile` to a file mounted at runtime. +- Cron jobs running on the same host can keep using the API thanks to the loopback + bypass mask. Add additional CIDR ranges as needed; every bypass is logged. +- Export the same configuration to Kubernetes or systemd by setting environment + variables such as: + + ```bash + export CONCELIER_AUTHORITY__ENABLED=true + export CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false + export CONCELIER_AUTHORITY__ISSUER="https://authority.example.org" + export CONCELIER_AUTHORITY__CLIENTID="concelier-jobs" + export CONCELIER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/concelier/authority-client" + export CONCELIER_AUTHORITY__REQUIREDSCOPES__0="concelier.jobs.trigger" + export CONCELIER_AUTHORITY__REQUIREDSCOPES__1="advisory:read" + export CONCELIER_AUTHORITY__REQUIREDSCOPES__2="advisory:ingest" + export CONCELIER_AUTHORITY__CLIENTSCOPES__0="concelier.jobs.trigger" + export CONCELIER_AUTHORITY__CLIENTSCOPES__1="advisory:read" + export CONCELIER_AUTHORITY__CLIENTSCOPES__2="advisory:ingest" + export CONCELIER_AUTHORITY__REQUIREDTENANTS__0="tenant-default" + ``` + - CLI commands already pass `Authorization` headers when credentials are supplied. Configure the CLI with matching Authority settings (`docs/09_API_CLI_REFERENCE.md`) so that automation can obtain tokens with the same client credentials. Concelier diff --git a/docs/11_AUTHORITY.md b/docs/11_AUTHORITY.md index 93b77d22..00cdd5c6 100644 --- a/docs/11_AUTHORITY.md +++ b/docs/11_AUTHORITY.md @@ -36,21 +36,59 @@ Authority persists every issued token in MongoDB so operators can audit or revok - **Collection:** `authority_tokens` - **Key fields:** - - `tokenId`, `type` (`access_token`, `refresh_token`, `device_code`, `authorization_code`) - - `subjectId`, `clientId`, ordered `scope` array - - `status` (`valid`, `revoked`, `expired`), `createdAt`, optional `expiresAt` - - `revokedAt`, machine-readable `revokedReason`, optional `revokedReasonDescription` - - `revokedMetadata` (string dictionary for plugin-specific context) -- **Persistence flow:** `PersistTokensHandler` stamps missing JWT IDs, normalises scopes, and stores every principal emitted by OpenIddict. -- **Revocation flow:** `AuthorityTokenStore.UpdateStatusAsync` flips status, records the reason metadata, and is invoked by token revocation handlers and plugin provisioning events (e.g., disabling a user). +- `tokenId`, `type` (`access_token`, `refresh_token`, `device_code`, `authorization_code`) +- `subjectId`, `clientId`, ordered `scope` array +- `tenant` (lower-cased tenant hint from the issuing client, omitted for global clients) +- `status` (`valid`, `revoked`, `expired`), `createdAt`, optional `expiresAt` +- `revokedAt`, machine-readable `revokedReason`, optional `revokedReasonDescription` +- `revokedMetadata` (string dictionary for plugin-specific context) +- **Persistence flow:** `PersistTokensHandler` stamps missing JWT IDs, normalises scopes, and stores every principal emitted by OpenIddict. +- **Revocation flow:** `AuthorityTokenStore.UpdateStatusAsync` flips status, records the reason metadata, and is invoked by token revocation handlers and plugin provisioning events (e.g., disabling a user). - **Expiry maintenance:** `AuthorityTokenStore.DeleteExpiredAsync` prunes non-revoked tokens past their `expiresAt` timestamp. Operators should schedule this in maintenance windows if large volumes of tokens are issued. ### Expectations for resource servers Resource servers (Concelier WebService, Backend, Agent) **must not** assume in-memory caches are authoritative. They should: - cache `/jwks` and `/revocations/export` responses within configured lifetimes; -- honour `revokedReason` metadata when shaping audit trails; -- treat `status != "valid"` or missing tokens as immediate denial conditions. +- honour `revokedReason` metadata when shaping audit trails; +- treat `status != "valid"` or missing tokens as immediate denial conditions. + +### Tenant propagation + +- Client provisioning (bootstrap or plug-in) accepts a `tenant` hint. Authority normalises the value (`trim().ToLowerInvariant()`) and persists it alongside the registration. Clients without an explicit tenant remain global. +- Issued principals include the `stellaops:tenant` claim. `PersistTokensHandler` mirrors this claim into `authority_tokens.tenant`, enabling per-tenant revocation and reporting. +- Rate limiter metadata now tags requests with `authority.tenant`, unlocking per-tenant throughput metrics and diagnostic filters. Audit events (`authority.client_credentials.grant`, `authority.password.grant`, bootstrap flows) surface the tenant and login attempt documents index on `{tenant, occurredAt}` for quick queries. +- Password grant flows reuse the client registration's tenant and enforce the configured scope allow-list. Requested scopes outside that list (or mismatched tenants) trigger `invalid_scope`/`invalid_client` failures, ensuring cross-tenant access is denied before token issuance. + +### Default service scopes + +| Client ID | Purpose | Scopes granted | Sender constraint | Tenant | +|----------------------|---------------------------------------|--------------------------------------|-------------------|-----------------| +| `concelier-ingest` | Concelier raw advisory ingestion | `advisory:ingest`, `advisory:read` | `dpop` | `tenant-default` | +| `excitor-ingest` | Excititor raw VEX ingestion | `vex:ingest`, `vex:read` | `dpop` | `tenant-default` | +| `aoc-verifier` | Aggregation-only contract verification | `aoc:verify` | `dpop` | `tenant-default` | +| `cartographer-service` | Graph snapshot construction | `graph:write`, `graph:read` | `dpop` | `tenant-default` | +| `graph-api` | Graph Explorer gateway/API | `graph:read`, `graph:export`, `graph:simulate` | `dpop` | `tenant-default` | +| `vuln-explorer-ui` | Vuln Explorer UI/API | `vuln:read` | `dpop` | `tenant-default` | + +> **Secret hygiene (2025‑10‑27):** The repository includes a convenience `etc/authority.yaml` for compose/helm smoke tests. Every entry’s `secretFile` points to `etc/secrets/*.secret`, which ship with `*-change-me` placeholders—replace them with strong values (and wire them through your vault/secret manager) before issuing tokens in CI, staging, or production. + +These registrations are provided as examples in `etc/authority.yaml.sample`. Clone them per tenant (for example `concelier-tenant-a`, `concelier-tenant-b`) so tokens remain tenant-scoped by construction. + +Graph Explorer introduces dedicated scopes: `graph:write` for Cartographer build jobs, `graph:read` for query/read operations, `graph:export` for long-running export downloads, and `graph:simulate` for what-if overlays. Assign only the scopes a client actually needs to preserve least privilege—UI-facing clients should typically request read/export access, while background services (Cartographer, Scheduler) require write privileges. + +#### Least-privilege guidance for graph clients + +- **Service identities** – The Cartographer worker should request `graph:write` and `graph:read` only; grant `graph:simulate` exclusively to pipeline automation that invokes Policy Engine overlays on demand. Keep `graph:export` scoped to API gateway components responsible for streaming GraphML/JSONL artifacts. Authority enforces this by rejecting `graph:write` tokens that lack `properties.serviceIdentity: cartographer`. +- **Tenant propagation** – Every client registration must pin a `tenant` hint. Authority normalises the value and stamps it into issued tokens (`stellaops:tenant`) so downstream services (Scheduler, Graph API, Console) can enforce tenant isolation without custom headers. Graph scopes (`graph:read`, `graph:write`, `graph:export`, `graph:simulate`) are denied if the tenant hint is missing. +- **SDK alignment** – Use the generated `StellaOpsScopes` constants in service code to request graph scopes. Hard-coded strings risk falling out of sync as additional graph capabilities are added. +- **DPOP for automation** – Maintain sender-constrained (`dpop`) flows for Cartographer and Scheduler to limit reuse of access tokens if a build host is compromised. For UI-facing tokens, pair `graph:read`/`graph:export` with short lifetimes and enforce refresh-token rotation at the gateway. + +#### Vuln Explorer permalinks + +- **Scope** – `vuln:read` authorises Vuln Explorer to fetch advisory/linkset evidence and issue shareable links. Assign it only to front-end/API clients that must render vulnerability details. +- **Signed links** – `POST /permalinks/vuln` (requires `vuln:read`) accepts `{ "tenant": "tenant-a", "resourceKind": "vulnerability", "state": { ... }, "expiresInSeconds": 86400 }` and returns a JWT (`token`) plus `issuedAt`/`expiresAt`. The token embeds the tenant, requested state, and `vuln:read` scope and is signed with the same Authority signing keys published via `/jwks`. +- **Validation** – Resource servers verify the permalink using cached JWKS: check signature, ensure the tenant matches the current request context, honour the expiry, and enforce the contained `vuln:read` scope. The payload’s `resource.state` block is opaque JSON so UIs can round-trip filters/search terms without new schema changes. ## 4. Revocation Pipeline Authority centralises revocation in `authority_revocations` with deterministic categories: @@ -119,18 +157,38 @@ Authority signs revocation bundles and publishes JWKS entries via the new signin The rotation API leverages the same cryptography abstractions as revocation signing; no restart is required and the previous key is marked `retired` but kept available for verification. ## 6. Bootstrap & Administrative Endpoints -Administrative APIs live under `/internal/*` and require the bootstrap API key plus rate-limiter compliance. - -| Endpoint | Method | Description | -| --- | --- | --- | -| `/internal/users` | `POST` | Provision initial administrative accounts through the registered password-capable plug-in. Emits structured audit events. | -| `/internal/clients` | `POST` | Provision OAuth clients (client credentials / device code). | -| `/internal/revocations/export` | `GET` | Export revocation bundle + detached JWS + digest. | -| `/internal/signing/rotate` | `POST` | Promote a new signing key (see SOP above). Request body accepts `keyId`, `location`, optional `source`, `algorithm`, `provider`, and metadata. | - -All administrative calls emit `AuthEventRecord` entries enriched with correlation IDs, PII tags, and network metadata for offline SOC ingestion. - -## 7. Configuration Reference +Administrative APIs live under `/internal/*` and require the bootstrap API key plus rate-limiter compliance. + +| Endpoint | Method | Description | +| --- | --- | --- | +| `/internal/users` | `POST` | Provision initial administrative accounts through the registered password-capable plug-in. Emits structured audit events. | +| `/internal/clients` | `POST` | Provision OAuth clients (client credentials / device code). | +| `/internal/revocations/export` | `GET` | Export revocation bundle + detached JWS + digest. | +| `/internal/signing/rotate` | `POST` | Promote a new signing key (see SOP above). Request body accepts `keyId`, `location`, optional `source`, `algorithm`, `provider`, and metadata. | + +All administrative calls emit `AuthEventRecord` entries enriched with correlation IDs, PII tags, and network metadata for offline SOC ingestion. + +> **Tenant hint:** include a `tenant` entry inside `properties` when bootstrapping clients. Authority normalises the value, stores it on the registration, and stamps future tokens/audit events with the tenant. + +### Bootstrap client example + +```jsonc +POST /internal/clients +{ + "clientId": "concelier", + "confidential": true, + "displayName": "Concelier Backend", + "allowedGrantTypes": ["client_credentials"], + "allowedScopes": ["concelier.jobs.trigger", "advisory:ingest", "advisory:read"], + "properties": { + "tenant": "tenant-default" + } +} +``` + +For environments with multiple tenants, repeat the call per tenant-specific client (e.g. `concelier-tenant-a`, `concelier-tenant-b`) or append suffixes to the client identifier. + +## 7. Configuration Reference | Section | Key | Description | Notes | | --- | --- | --- | --- | @@ -181,11 +239,45 @@ Authority now understands two flavours of sender-constrained OAuth clients: - Certificate bindings now act as an allow-list: Authority verifies thumbprint, subject, issuer, serial number, and any declared SAN values against the presented certificate, with rotation grace windows applied to `notBefore/notAfter`. Operators can enforce subject regexes, SAN type allow-lists (`dns`, `uri`, `ip`), trusted certificate authorities, and rotation grace via `security.senderConstraints.mtls.*`. Both modes persist additional metadata in `authority_tokens`: `senderConstraint` records the enforced policy, while `senderKeyThumbprint` stores the DPoP JWK thumbprint or mTLS certificate hash captured at issuance. Downstream services can rely on these fields (and the corresponding `cnf` claim) when auditing offline copies of the token store. - -## 8. Offline & Sovereign Operation -- **No outbound dependencies:** Authority only contacts MongoDB and local plugins. Discovery and JWKS are cached by clients with offline tolerances (`AllowOfflineCacheFallback`, `OfflineCacheTolerance`). Operators should mirror these responses for air-gapped use. -- **Structured logging:** Every revocation export, signing rotation, bootstrap action, and token issuance emits structured logs with `traceId`, `client_id`, `subjectId`, and `network.remoteIp` where applicable. Mirror logs to your SIEM to retain audit trails without central connectivity. -- **Determinism:** Sorting rules in token and revocation exports guarantee byte-for-byte identical artefacts given the same datastore state. Hashes and signatures remain stable across machines. + +### 7.2 Policy Engine clients & scopes + +Policy Engine v2 introduces dedicated scopes and a service identity that materialises effective findings. Configure Authority as follows when provisioning policy clients: + +| Client | Scopes | Notes | +| --- | --- | --- | +| `policy-engine` (service) | `policy:run`, `findings:read`, `effective:write` | Must include `properties.serviceIdentity: policy-engine` and a tenant. Authority rejects `effective:write` tokens without the marker or tenant. | +| `policy-cli` / automation | `policy:write`, `policy:submit`, `policy:run`, `findings:read` | Keep scopes minimal; only trusted automation should add `policy:approve`/`policy:activate`. | +| UI/editor sessions | `policy:read`, `policy:write`, `policy:simulate` (+ reviewer/approver scopes as appropriate) | Issue tenant-specific clients so audit and rate limits remain scoped. | + +Sample YAML entry: + +```yaml + - clientId: "policy-engine" + displayName: "Policy Engine Service" + grantTypes: [ "client_credentials" ] + audiences: [ "api://policy-engine" ] + scopes: [ "policy:run", "findings:read", "effective:write" ] + tenant: "tenant-default" + properties: + serviceIdentity: "policy-engine" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/policy-engine.secret" +``` + +Compliance checklist: + +- [ ] `policy-engine` client includes `properties.serviceIdentity: policy-engine` and a tenant hint; logins missing either are rejected. +- [ ] Non-service clients omit `effective:write` and receive only the scopes required for their role (`policy:write`, `policy:submit`, `policy:approve`, `policy:activate`, etc.). +- [ ] Approval/activation workflows use identities distinct from authoring identities; tenants are provisioned per client to keep telemetry segregated. +- [ ] Operators document reviewer assignments and incident procedures alongside `/docs/security/policy-governance.md` and archive policy evidence bundles (`stella policy bundle export`) with each release. + +## 8. Offline & Sovereign Operation +- **No outbound dependencies:** Authority only contacts MongoDB and local plugins. Discovery and JWKS are cached by clients with offline tolerances (`AllowOfflineCacheFallback`, `OfflineCacheTolerance`). Operators should mirror these responses for air-gapped use. +- **Structured logging:** Every revocation export, signing rotation, bootstrap action, and token issuance emits structured logs with `traceId`, `client_id`, `subjectId`, and `network.remoteIp` where applicable. Mirror logs to your SIEM to retain audit trails without central connectivity. +- **Determinism:** Sorting rules in token and revocation exports guarantee byte-for-byte identical artefacts given the same datastore state. Hashes and signatures remain stable across machines. ## 9. Operational Checklist - [ ] Protect the bootstrap API key and disable bootstrap endpoints (`bootstrap.enabled: false`) once initial setup is complete. diff --git a/docs/11_DATA_SCHEMAS.md b/docs/11_DATA_SCHEMAS.md index e99c1654..0e98e14e 100755 --- a/docs/11_DATA_SCHEMAS.md +++ b/docs/11_DATA_SCHEMAS.md @@ -232,6 +232,101 @@ Images are deduplicated and sorted by digest. Label keys are normalised to lower ``` Metadata keys are lowercased, first‑writer wins (duplicates with different casing are ignored), and optional IDs (`scheduleId`, `runId`) are trimmed when empty. Use the canonical serializer when emitting events so audit digests remain reproducible. + +#### 3.1.5 Run Summary (`run_summaries`) + +Materialized view powering the Scheduler UI dashboards. Stores the latest roll-up per schedule/tenant, enabling quick “last run” banners and sparkline counters without scanning the full `runs` collection. + +```jsonc +{ + "tenantId": "tenant-alpha", + "scheduleId": "sch_20251018a", + "updatedAt": "2025-10-18T22:10:10Z", + "lastRun": { + "runId": "run_20251018_0001", + "trigger": "feedser", + "state": "completed", + "createdAt": "2025-10-18T22:03:14Z", + "startedAt": "2025-10-18T22:03:20Z", + "finishedAt": "2025-10-18T22:08:45Z", + "stats": { + "candidates": 1280, + "deduped": 910, + "queued": 0, + "completed": 910, + "deltas": 42, + "newCriticals": 7, + "newHigh": 11, + "newMedium": 18, + "newLow": 6 + }, + "error": null + }, + "recent": [ + { + "runId": "run_20251018_0001", + "trigger": "feedser", + "state": "completed", + "createdAt": "2025-10-18T22:03:14Z", + "startedAt": "2025-10-18T22:03:20Z", + "finishedAt": "2025-10-18T22:08:45Z", + "stats": { + "candidates": 1280, + "deduped": 910, + "queued": 0, + "completed": 910, + "deltas": 42, + "newCriticals": 7, + "newHigh": 11, + "newMedium": 18, + "newLow": 6 + }, + "error": null + }, + { + "runId": "run_20251017_0003", + "trigger": "cron", + "state": "error", + "createdAt": "2025-10-17T22:01:02Z", + "startedAt": "2025-10-17T22:01:08Z", + "finishedAt": "2025-10-17T22:04:11Z", + "stats": { + "candidates": 1040, + "deduped": 812, + "queued": 0, + "completed": 640, + "deltas": 18, + "newCriticals": 2, + "newHigh": 4, + "newMedium": 7, + "newLow": 3 + }, + "error": "scanner timeout" + } + ], + "counters": { + "total": 3, + "planning": 0, + "queued": 0, + "running": 0, + "completed": 1, + "error": 1, + "cancelled": 1, + "totalDeltas": 60, + "totalNewCriticals": 9, + "totalNewHigh": 15, + "totalNewMedium": 25, + "totalNewLow": 9 + } +} +``` + +- `_id` combines `tenantId` and `scheduleId` (`tenant:schedule`). +- `recent` contains the 20 most recent runs ordered by `createdAt` (UTC). Updates replace the existing entry for a run to respect state transitions. +- `counters` aggregate over the retained window (20 runs) for quick trend indicators. Totals are recomputed after every update. +- Schedulers should call the projection service after every run state change so the cache mirrors planner/runner progress. + +Sample file: `samples/api/scheduler/run-summary.json`. --- diff --git a/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md b/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md index 5ba34c27..2221c4dc 100755 --- a/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md +++ b/docs/13_RELEASE_ENGINEERING_PLAYBOOK.md @@ -65,6 +65,9 @@ graph LR `ops/devops/release/build_release.py` to build multi-arch images, attach CycloneDX SBOMs and SLSA provenance with Cosign, and emit `out/release/release.yaml` for downstream packaging (Helm, Compose, Offline Kit). +The `build-test-deploy` workflow also runs +`python ops/devops/release/test_verify_release.py` so release verifier +regressions fail fast during every CI pass. --- @@ -100,11 +103,12 @@ ouk fetch \ --sign cosign.key ``` -### 4.2 Pipeline Hook - -* Runs on **first Friday** each month (cron). -* Generates tarball, signs it, uploads to **GitLab Release asset**. -* SHA‑256 + signature published alongside. +### 4.2 Pipeline Hook + +* Runs on **first Friday** each month (cron). +* Generates tarball, signs it, uploads to **GitLab Release asset**. +* SHA‑256 + signature published alongside. +* Release job must emit `out/release/debug/` with `debug-manifest.json` and `.sha256` so `ops/offline-kit/mirror_debug_store.py` can mirror symbols into the Offline Kit (see `DEVOPS-REL-17-004`). ### 4.3 Activation Flow (runtime) @@ -123,12 +127,13 @@ CI job fails if token expiry < 29 days (guard against stale caches). ## 5 Artifact Signing & Transparency -| Artefact | Signer | Tool | -| ------------ | --------------- | --------------------- | -| Git tags | GPG (`0x90C4…`) | `git tag -s` | -| Containers | Cosign key pair | `cosign sign` | -| Helm Charts | prov file | `helm package --sign` | -| OUK tarballs | Cosign | `cosign sign-blob` | +| Artefact | Signer | Tool/Notes | +| ------------ | --------------- | ---------------------------------- | +| Git tags | GPG (`0x90C4…`) | `git tag -s` | +| Containers | Cosign key pair | `cosign sign` | +| Helm Charts | prov file | `helm package --sign` | +| OUK tarballs | Cosign | `cosign sign-blob` | +| Debug store | — | `debug/debug-manifest.json` hashed | **Rekor** integration is **TODO** – once the internal Rekor mirror is online (`StellaOpsAttestor`) a post‑publish job will submit transparency log entries. @@ -141,9 +146,20 @@ CI job fails if token expiry < 29 days (guard against stale caches). 3. Tag `git tag -s X.Y.Z -m "Release X.Y.Z"` & push. 4. GitLab CI auto‑publishes images & charts. 5. Draft GitLab **Release Notes** using `tools/release-notes-gen`. -6. Verify SBOM attachment with `stella sbom verify stella/backend:X.Y.Z`. -7. Smoke‑test OUK tarball in offline lab. -8. Announce in `#stella-release` Mattermost channel. +6. Verify SBOM attachment with `stella sbom verify stella/backend:X.Y.Z`. +7. Run the release verifier locally if CI isn’t available (mirrors the workflow step): + `python ops/devops/release/test_verify_release.py` +8. Mirror the release debug store into the Offline Kit staging tree and re-check the manifest: + ```bash + ./ops/offline-kit/mirror_debug_store.py \ + --release-dir out/release \ + --offline-kit-dir out/offline-kit + jq '.artifacts | length' out/offline-kit/debug/debug-manifest.json + readelf -n /app/... | grep -i 'Build ID' + ``` + Validate that the hash from `readelf` matches the `.build-id//.debug` path created by the script. +9. Smoke-test OUK tarball in offline lab. +10. Announce in `#stella-release` Mattermost channel. --- diff --git a/docs/21_INSTALL_GUIDE.md b/docs/21_INSTALL_GUIDE.md index 5667bb31..a6f8c830 100755 --- a/docs/21_INSTALL_GUIDE.md +++ b/docs/21_INSTALL_GUIDE.md @@ -1,70 +1,70 @@ -# Stella Ops — Installation Guide (Docker & Air‑Gap) - - - -> **Status — public α not yet published.** -> The commands below will work as soon as the first image is tagged -> `registry.stella-ops.org/stella-ops/stella-ops:0.1.0-alpha` -> (target date: **late 2025**). Track progress on the -> [road‑map](/roadmap/). - ---- - -## 0 · Prerequisites - -| Item | Minimum | Notes | -|------|---------|-------| -| Linux | Ubuntu 22.04 LTS / Alma 9 | x86‑64 or arm64 | -| CPU / RAM | 2 vCPU / 2 GiB | Laptop baseline | -| Disk | 10 GiB SSD | SBOM + vuln DB cache | -| Docker | **Engine 25 + Compose v2** | `docker -v` | -| TLS | OpenSSL 1.1 +  | Self‑signed cert generated at first run | - ---- - -## 1 · Connected‑host install (Docker Compose) - -```bash -# 1. Make a working directory -mkdir stella && cd stella - -# 2. Download the signed Compose bundle + example .env -curl -LO https://get.stella-ops.org/releases/latest/.env.example -curl -LO https://get.stella-ops.org/releases/latest/.env.example.sig -curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml -curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml.sig -curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml -curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml.sig - -# 3. Verify provenance (Cosign public key is stable) -cosign verify-blob \ - --key https://stella-ops.org/keys/cosign.pub \ - --signature .env.example.sig \ - .env.example - -cosign verify-blob \ - --key https://stella-ops.org/keys/cosign.pub \ - --signature docker-compose.infrastructure.yml.sig \ - docker-compose.infrastructure.yml - -cosign verify-blob \ - --key https://stella-ops.org/keys/cosign.pub \ - --signature docker-compose.stella-ops.yml.sig \ - docker-compose.stella-ops.yml - -# 4. Copy .env.example → .env and edit secrets -cp .env.example .env -$EDITOR .env - -# 5. Launch databases (MongoDB + Redis) -docker compose --env-file .env -f docker-compose.infrastructure.yml up -d - +# Stella Ops — Installation Guide (Docker & Air‑Gap) + + + +> **Status — public α not yet published.** +> The commands below will work as soon as the first image is tagged +> `registry.stella-ops.org/stella-ops/stella-ops:0.1.0-alpha` +> (target date: **late 2025**). Track progress on the +> [road‑map](/roadmap/). + +--- + +## 0 · Prerequisites + +| Item | Minimum | Notes | +|------|---------|-------| +| Linux | Ubuntu 22.04 LTS / Alma 9 | x86‑64 or arm64 | +| CPU / RAM | 2 vCPU / 2 GiB | Laptop baseline | +| Disk | 10 GiB SSD | SBOM + vuln DB cache | +| Docker | **Engine 25 + Compose v2** | `docker -v` | +| TLS | OpenSSL 1.1 +  | Self‑signed cert generated at first run | + +--- + +## 1 · Connected‑host install (Docker Compose) + +```bash +# 1. Make a working directory +mkdir stella && cd stella + +# 2. Download the signed Compose bundle + example .env +curl -LO https://get.stella-ops.org/releases/latest/.env.example +curl -LO https://get.stella-ops.org/releases/latest/.env.example.sig +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.infrastructure.yml.sig +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml +curl -LO https://get.stella-ops.org/releases/latest/docker-compose.stella-ops.yml.sig + +# 3. Verify provenance (Cosign public key is stable) +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature .env.example.sig \ + .env.example + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.infrastructure.yml.sig \ + docker-compose.infrastructure.yml + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature docker-compose.stella-ops.yml.sig \ + docker-compose.stella-ops.yml + +# 4. Copy .env.example → .env and edit secrets +cp .env.example .env +$EDITOR .env + +# 5. Launch databases (MongoDB + Redis) +docker compose --env-file .env -f docker-compose.infrastructure.yml up -d + # 6. Launch Stella Ops (first run pulls ~50 MB merged vuln DB) docker compose --env-file .env -f docker-compose.stella-ops.yml up -d ```` @@ -95,7 +95,13 @@ The Concelier container reads configuration from `etc/concelier.yaml` plus CONCELIER_AUTHORITY__ISSUER="https://authority.internal" CONCELIER_AUTHORITY__AUDIENCES__0="api://concelier" CONCELIER_AUTHORITY__REQUIREDSCOPES__0="concelier.jobs.trigger" + CONCELIER_AUTHORITY__REQUIREDSCOPES__1="advisory:read" + CONCELIER_AUTHORITY__REQUIREDSCOPES__2="advisory:ingest" + CONCELIER_AUTHORITY__REQUIREDTENANTS__0="tenant-default" CONCELIER_AUTHORITY__CLIENTID="concelier-jobs" + CONCELIER_AUTHORITY__CLIENTSCOPES__0="concelier.jobs.trigger" + CONCELIER_AUTHORITY__CLIENTSCOPES__1="advisory:read" + CONCELIER_AUTHORITY__CLIENTSCOPES__2="advisory:ingest" CONCELIER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/concelier_authority_client" CONCELIER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32" CONCELIER_AUTHORITY__BYPASSNETWORKS__1="::1/128" @@ -132,53 +138,53 @@ The Concelier container reads configuration from `etc/concelier.yaml` plus --- ## 2 · Optional: request a free quota token - -Anonymous installs allow **{{ quota\_anon }} scans per UTC day**. -Email `token@stella-ops.org` to receive a signed JWT that raises the limit to -**{{ quota\_token }} scans/day**. Insert it into `.env`: - -```bash -STELLA_JWT="paste‑token‑here" -docker compose --env-file .env -f docker-compose.stella-ops.yml \ - exec stella-ops stella set-jwt "$STELLA_JWT" -``` - ->  The UI shows a reminder at 200 scans and throttles above the limit but will ->  **never block** your pipeline. - ---- - -## 3 · Air‑gapped install (Offline Update Kit) - -When running on an isolated network use the **Offline Update Kit (OUK)**: - -```bash -# Download & verify on a connected host -curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz -curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz.sig - -cosign verify-blob \ - --key https://stella-ops.org/keys/cosign.pub \ - --signature stella-ops-offline-kit-v0.1a.tgz.sig \ - stella-ops-offline-kit-v0.1a.tgz - -# Transfer → air‑gap → import -docker compose --env-file .env -f docker-compose.stella-ops.yml \ - exec stella admin import-offline-usage-kit stella-ops-offline-kit-v0.1a.tgz -``` - -*Import is atomic; no service downtime.* - -For details see the dedicated [Offline Kit guide](/offline/). - ---- - -## 4 · Next steps - -* **5‑min Quick‑Start:** `/quickstart/` -* **CI recipes:** `docs/ci/20_CI_RECIPES.md` -* **Plug‑in SDK:** `/plugins/` - ---- - -*Generated {{ "now" | date: "%Y‑%m‑%d" }} — build tags inserted at render time.* + +Anonymous installs allow **{{ quota\_anon }} scans per UTC day**. +Email `token@stella-ops.org` to receive a signed JWT that raises the limit to +**{{ quota\_token }} scans/day**. Insert it into `.env`: + +```bash +STELLA_JWT="paste‑token‑here" +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella-ops stella set-jwt "$STELLA_JWT" +``` + +>  The UI shows a reminder at 200 scans and throttles above the limit but will +>  **never block** your pipeline. + +--- + +## 3 · Air‑gapped install (Offline Update Kit) + +When running on an isolated network use the **Offline Update Kit (OUK)**: + +```bash +# Download & verify on a connected host +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz +curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-v0.1a.tgz.sig + +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-kit-v0.1a.tgz.sig \ + stella-ops-offline-kit-v0.1a.tgz + +# Transfer → air‑gap → import +docker compose --env-file .env -f docker-compose.stella-ops.yml \ + exec stella admin import-offline-usage-kit stella-ops-offline-kit-v0.1a.tgz +``` + +*Import is atomic; no service downtime.* + +For details see the dedicated [Offline Kit guide](/offline/). + +--- + +## 4 · Next steps + +* **5‑min Quick‑Start:** `/quickstart/` +* **CI recipes:** `docs/ci/20_CI_RECIPES.md` +* **Plug‑in SDK:** `/plugins/` + +--- + +*Generated {{ "now" | date: "%Y‑%m‑%d" }} — build tags inserted at render time.* diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md index bbe6a42f..1c48a821 100755 --- a/docs/24_OFFLINE_KIT.md +++ b/docs/24_OFFLINE_KIT.md @@ -18,6 +18,8 @@ completely isolated network: | **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | | **Delta patches** | Daily diff bundles keep size \< 350 MB | | **Scanner plug-ins** | OS analyzers plus the Node.js, Go, .NET, and Python language analyzers packaged under `plugins/scanner/analyzers/**` with manifests so Workers load deterministically offline. | +| **Debug store** | `.debug` artefacts laid out under `debug/.build-id//.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. | +| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. | **RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an air‑gapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache. @@ -25,11 +27,53 @@ completely isolated network: *Scanner core:* C# 12 on **.NET {{ dotnet }}**. *Imports are idempotent and atomic — no service downtime.* - ---- - -## 1 · Download & verify - + +## 0 · Prepare the debug store + +Before packaging the Offline Kit, mirror the release debug artefacts (GNU build-id `.debug` files and the associated manifest) into the staging directory: + +```bash +./ops/offline-kit/mirror_debug_store.py \ + --release-dir out/release \ + --offline-kit-dir out/offline-kit +``` + +The helper copies `debug/.build-id/**`, validates `debug/debug-manifest.json` against its recorded SHA-256, and writes `out/offline-kit/metadata/debug-store.json` with a short summary (platforms, artefact counts, sample build-ids). The command exits non-zero if an artefact referenced by the manifest is missing or has the wrong digest, so run it as part of every kit build. + +--- + +## 0.1 · Automated packaging + +The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`. +It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set. + +```bash +python ops/offline-kit/build_offline_kit.py \ + --version 2025.10.0 \ + --channel edge \ + --release-dir out/release \ + --staging-dir out/offline-kit/staging \ + --output-dir out/offline-kit/dist + +# Optional: regenerate the telemetry collector bundle prior to packaging. +python ops/devops/telemetry/package_offline_bundle.py --output out/telemetry/telemetry-offline-bundle.tar.gz +``` + +Outputs: + +- `stella-ops-offline-kit--.tar.gz` — bundle (mtime/uid/gid forced to zero for reproducibility) +- `stella-ops-offline-kit--.tar.gz.sha256` — bundle digest +- `manifest/offline-manifest.json` + `.sha256` — inventories every file in the bundle +- `.metadata.json` — descriptor consumed by the CLI/Console import tooling +- `telemetry/telemetry-offline-bundle.tar.gz` + `.sha256` — packaged OTLP collector assets for environments without upstream access +- `plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/*.sig` (+ `.sha256`) — Cosign signatures for the Python analyzer DLL and manifest + +Provide `--cosign-key` / `--cosign-identity-token` (and optional `--cosign-password`) to generate Cosign signatures for both the tarball and manifest. + +--- + +## 1 · Download & verify + ```bash curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-.tgz curl -LO https://get.stella-ops.org/ouk/stella-ops-offline-kit-.tgz.sig @@ -101,21 +145,21 @@ Example excerpt (2025-10-23 kit) showing the Go and .NET analyzer plug-in payloa } { "name": "plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.dll", - "sha256": "28b6e06c7cabf3b78f13f801cbb14962093f3d42c4ae9ec01babbcd14cda4644", - "size": 53760, - "capturedAt": "2025-10-23T00:00:00Z" + "sha256": "a4f558f363394096e3dd6263f35b180b93b4112f9cf616c05872da8a8657d518", + "size": 47104, + "capturedAt": "2025-10-26T00:00:00Z" } { "name": "plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.pdb", - "sha256": "be4e34b4dc9a790fe1299e84213343b7c8ea90a2d22e5d7d1aa7585b8fedc946", - "size": 34516, - "capturedAt": "2025-10-23T00:00:00Z" + "sha256": "ef2ad78bc2cd1d7e99bae000b92357aa9a9c32938501899e9033d001096196d0", + "size": 31896, + "capturedAt": "2025-10-26T00:00:00Z" } { "name": "plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/manifest.json", - "sha256": "bceea1e7542aae860b0ec5ba7b8b3aa960b21edc4d1efe60afc98ce289341ac3", - "size": 671, - "capturedAt": "2025-10-23T00:00:00Z" + "sha256": "668ad9a1a35485628677b639db4d996d1e25f62021680a81a22482483800e557", + "size": 648, + "capturedAt": "2025-10-26T00:00:00Z" } ``` @@ -153,6 +197,21 @@ tar -tzf stella-ops-offline-kit-.tgz 'plugins/scanner/analyzers/lang/Stell The manifest lookup above and this `tar` listing should both surface the Go analyzer DLL, PDB, and manifest entries before the kit is promoted. +> **Release guardrail.** The automated release pipeline now publishes the Python plug-in from source and executes `dotnet run --project tools/LanguageAnalyzerSmoke --configuration Release -- --repo-root ` to validate manifest integrity and cold/warm determinism within the < 30 s / < 5 s budgets (differences versus repository goldens are logged for triage). Run `ops/offline-kit/run-python-analyzer-smoke.sh` locally before shipping a refreshed kit if you rebuild artefacts outside CI or when preparing the air-gap bundle. + +### Debug store mirror + +Offline symbols (`debug/.build-id/**`) must accompany every Offline Kit to keep symbol lookup deterministic. The release workflow is expected to emit `out/release/debug/` containing the build-id tree plus `debug-manifest.json` and its `.sha256` companion. After a release completes: + +```bash +python ops/offline-kit/mirror_debug_store.py \ + --release-dir out/release \ + --offline-dir out/offline-kit \ + --summary out/offline-kit/metadata/debug-store.json +``` + +The script mirrors the debug tree into the Offline Kit staging directory, verifies SHA-256 values against the manifest, and writes a summary under `metadata/debug-store.json` for audit logs. If the release pipeline does not populate `out/release/debug`, the tooling now logs a warning (`DEVOPS-REL-17-004`)—treat it as a build failure and re-run the release once symbol extraction is enabled. + --- ## 3 · Delta patch workflow diff --git a/docs/ARCHITECTURE_CONCELIER.md b/docs/ARCHITECTURE_CONCELIER.md index cd841b76..c70369fa 100644 --- a/docs/ARCHITECTURE_CONCELIER.md +++ b/docs/ARCHITECTURE_CONCELIER.md @@ -123,6 +123,23 @@ details // structured conflict explanation / merge reasoning - Conflict explainers are serialized as deterministic `MergeConflictExplainerPayload` records (type, reason, source ranks, winning values); replay clients can parse the payload to render human-readable rationales without re-computing precedence. - Concelier.WebService exposes the immutable log via `GET /concelier/advisories/{vulnerabilityKey}/replay[?asOf=UTC_ISO8601]`, returning the latest statements (with hex-encoded hashes) and any conflict explanations for downstream exporters and APIs. +**AdvisoryObservation (new in Sprint 24)** + +``` +observationId // deterministic id: {tenant}:{source}:{upstreamId}:{revision} +tenant // issuing tenant (lower-case) +source{vendor,stream,api,collectorVersion} +upstream{ + upstreamId, documentVersion, contentHash, + fetchedAt, receivedAt, signature{present,format,keyId,signature}} +content{format,specVersion,raw,metadata} +linkset{aliases[], purls[], cpes[], references[{type,url}]} +createdAt // when Concelier recorded the observation +attributes // optional provenance metadata (e.g., batch, connector) +``` + +The observation is an immutable projection of the raw ingestion document (post provenance validation, pre-merge) that powers Link‑Not‑Merge overlays and Vuln Explorer. Observations live in the `advisory_observations` collection, keyed by tenant + upstream identity. `linkset` provides normalized aliases/PURLs/CPES that downstream services (Graph/Vuln Explorer) join against without triggering merge logic. Concelier.Core exposes strongly-typed models (`AdvisoryObservation`, `AdvisoryObservationLinkset`, etc.) and a Mongo-backed store for filtered queries by tenant/alias; this keeps overlay consumers read-only while preserving AOC guarantees. + **ExportState** ``` diff --git a/docs/ARCHITECTURE_DEVOPS.md b/docs/ARCHITECTURE_DEVOPS.md index 919de2c1..eddb74d0 100644 --- a/docs/ARCHITECTURE_DEVOPS.md +++ b/docs/ARCHITECTURE_DEVOPS.md @@ -76,8 +76,14 @@ At startup, services **self‑advertise** their semver & channel; the UI surface * **Unit/integration**: per‑component, plus **end‑to‑end** flows (scan→vex→policy→sign→attest). * **Perf SLOs**: hot paths (SBOM compose, diff, export) measured against budgets. * **Security**: dependency audit vs Concelier export; container hardening tests; minimal caps. +* **Analyzer smoke**: restart-time language plug-ins (currently Python) verified via `dotnet run --project tools/LanguageAnalyzerSmoke` to ensure manifest integrity plus cold vs warm determinism (< 30 s / < 5 s budgets); the harness logs deviations from repository goldens for follow-up. * **Canary cohort**: internal staging + selected customers; one week on **edge** before **stable** tag. +### 2.5 Debug-store artefacts + +* Every release exports stripped debug information for ELF binaries discovered in service images. Debug files follow the GNU build-id layout (`debug/.build-id//.debug`) and are generated via `objcopy --only-keep-debug`. +* `debug/debug-manifest.json` captures build-id → component/image/source mappings with SHA-256 checksums so operators can mirror the directory into debuginfod or offline symbol stores. The manifest (and its `.sha256` companion) ships with every release bundle and Offline Kit. + --- ## 3) Distribution & activation @@ -92,7 +98,7 @@ At startup, services **self‑advertise** their semver & channel; the UI surface **Gating policy**: * **Core images** (Authority, Scanner, Concelier, Excititor, Attestor, UI): public **read**. -* **Enterprise add‑ons** (if any) and **pre‑release**: private repos via OAuth2 token service. +* **Enterprise add‑ons** (if any) and **pre‑release**: private repos via the **Registry Token Service** (`src/StellaOps.Registry.TokenService`) which exchanges Authority-issued OpToks for short-lived Docker registry bearer tokens. > Monetization lever is **signing** (PoE gate), not image pulls, so the core remains simple to consume. @@ -105,6 +111,8 @@ At startup, services **self‑advertise** their semver & channel; the UI surface 3. Registry allows pull for the requested repo. * Tokens are **short‑lived** (60–300 s) and **DPoP‑bound**. +The token service enforces plan gating via `registry-token.yaml` (see `docs/ops/registry-token-service.md`) and exposes Prometheus metrics (`registry_token_issued_total`, `registry_token_rejected_total`). Revoked licence identifiers halt issuance even when scope requirements are met. + ### 3.3 Offline kits (air‑gapped) * Tarball per release channel: diff --git a/docs/ARCHITECTURE_SCANNER.md b/docs/ARCHITECTURE_SCANNER.md index d1202068..8d9df7ed 100644 --- a/docs/ARCHITECTURE_SCANNER.md +++ b/docs/ARCHITECTURE_SCANNER.md @@ -242,12 +242,15 @@ When `scanner.events.enabled = true`, the WebService serialises the signed repor ### 5.5 SBOM assembly & emit -* **Per‑layer SBOM fragments**: components introduced by the layer (+ relationships). +* **Per-layer SBOM fragments**: components introduced by the layer (+ relationships). * **Image SBOMs**: merge fragments; refer back to them via **CycloneDX BOM‑Link** (or SPDX ExternalRef). * Emit both **Inventory** & **Usage** views. +* When the native analyzer reports an ELF `buildId`, attach it to component metadata and surface it as `stellaops:buildId` in CycloneDX properties (and diff metadata). This keeps SBOM/diff output in lockstep with runtime events and the debug-store manifest. * Serialize **CycloneDX JSON** and **CycloneDX Protobuf**; optionally **SPDX 3.0.1 JSON**. * Build **BOM‑Index** sidecar: purl table + roaring bitmap; flag `usedByEntrypoint` components for fast backend joins. +The emitted `buildId` metadata is preserved in component hashes, diff payloads, and `/policy/runtime` responses so operators can pivot from SBOM entries → runtime events → `debug/.build-id//.debug` within the Offline Kit or release bundle. + ### 5.6 DSSE attestation (via Signer/Attestor) * WebService constructs **predicate** with `image_digest`, `stellaops_version`, `license_id`, `policy_digest?` (when emitting **final reports**), timestamps. diff --git a/docs/ARCHITECTURE_ZASTAVA.md b/docs/ARCHITECTURE_ZASTAVA.md index e90852e3..4bee85e9 100644 --- a/docs/ARCHITECTURE_ZASTAVA.md +++ b/docs/ARCHITECTURE_ZASTAVA.md @@ -141,10 +141,10 @@ stellaops/zastava-agent # System service; watch Docker events; observer on * Image signature presence (if cosign policies are local; else ask backend). * SBOM **referrers** presence (HEAD to registry, optional). * Rekor UUID known (query Scanner.WebService by image digest). -* **Publish runtime events** to Scanner.WebService `/runtime/events` (batch & compress). -* **Request delta scan** if: no SBOM in catalog OR base differs from known baseline. - -### 3.2 Privileges & mounts (K8s) +* **Publish runtime events** to Scanner.WebService `/runtime/events` (batch & compress). +* **Request delta scan** if: no SBOM in catalog OR base differs from known baseline. + +### 3.2 Privileges & mounts (K8s) * **SecurityContext:** `runAsUser: 0`, `readOnlyRootFilesystem: true`, `allowPrivilegeEscalation: false`. * **Capabilities:** `CAP_SYS_PTRACE` (optional if using nsenter trace), `CAP_DAC_READ_SEARCH`. @@ -154,12 +154,22 @@ stellaops/zastava-agent # System service; watch Docker events; observer on * `/run/containerd/containerd.sock` (or CRI‑O socket) * `/var/lib/containerd/io.containerd.runtime.v2.task` (rootfs paths & pids) * **Networking:** cluster‑internal egress to Scanner.WebService only. -* **Rate limits:** hard caps for bytes hashed and file count per container to avoid noisy tenants. - -### 3.3 Event batching - -* Buffer ND‑JSON; flush by **N events** or **2 s**. -* Backpressure: local disk ring buffer (50 MB default) if Scanner is temporarily unavailable; drop oldest after cap with **metrics** and **warning** event. +* **Rate limits:** hard caps for bytes hashed and file count per container to avoid noisy tenants. + +### 3.3 Event batching + +* Buffer ND‑JSON; flush by **N events** or **2 s**. +* Backpressure: local disk ring buffer (50 MB default) if Scanner is temporarily unavailable; drop oldest after cap with **metrics** and **warning** event. + +### 3.4 Build-id capture & validation workflow + +1. When Observer sees a `CONTAINER_START` it dereferences `/proc//exe`, extracts the `NT_GNU_BUILD_ID` note, normalises it to lower-case hex, and sends it as `process.buildId` in the runtime envelope. +2. Scanner.WebService persists the observation and propagates the most recent hashes into `/policy/runtime` responses (`buildIds` list) and policy caches consumed by the webhook/CLI. +3. Release engineering copies the matching `.debug` files into the bundle (`debug/.build-id//.debug`) and publishes `debug/debug-manifest.json` with per-hash digests. Offline Kit packaging reuses those artefacts verbatim (see `ops/offline-kit/mirror_debug_store.py`). +4. Operators resolve symbols by either: + * calling `stellaops-cli runtime policy test --image ` to read the current `buildIds` and then fetching the corresponding `.debug` file from the bundle/offline mirror, or + * piping the hash into `debuginfod-find debuginfo ` when a `debuginfod` service is wired against the mirrored tree. +5. Missing hashes indicate stripped binaries without GNU notes; operators should trigger a rebuild with `-Wl,--build-id` or register a fallback symbol package as described in the runtime operations runbook. --- diff --git a/docs/README.md b/docs/README.md index 2585b387..3bd6f433 100755 --- a/docs/README.md +++ b/docs/README.md @@ -36,6 +36,7 @@ Everything here is open‑source and versioned — when you check out a git ta - **07 – [High‑Level Architecture](07_HIGH_LEVEL_ARCHITECTURE.md)** - **08 – [Architecture Decision Records](adr/index.md)** - **08 – Module Architecture Dossiers** + - [Architecture Overview](architecture/overview.md) - [Scanner](ARCHITECTURE_SCANNER.md) - [Concelier](ARCHITECTURE_CONCELIER.md) - [Excititor](ARCHITECTURE_EXCITITOR.md) @@ -43,6 +44,7 @@ Everything here is open‑source and versioned — when you check out a git ta - [Signer](ARCHITECTURE_SIGNER.md) - [Attestor](ARCHITECTURE_ATTESTOR.md) - [Authority](ARCHITECTURE_AUTHORITY.md) + - [Policy Engine](architecture/policy-engine.md) - [Notify](ARCHITECTURE_NOTIFY.md) - [Scheduler](ARCHITECTURE_SCHEDULER.md) - [CLI](ARCHITECTURE_CLI.md) @@ -55,17 +57,32 @@ Everything here is open‑source and versioned — when you check out a git ta - **10 – [BuildX Generator Quickstart](dev/BUILDX_PLUGIN_QUICKSTART.md)** - **10 – [Scanner Cache Configuration](dev/SCANNER_CACHE_CONFIGURATION.md)** - **30 – [Excititor Connector Packaging Guide](dev/30_EXCITITOR_CONNECTOR_GUIDE.md)** +- **31 – [Aggregation-Only Contract Reference](ingestion/aggregation-only-contract.md)** - **30 – Developer Templates** - [Excititor Connector Skeleton](dev/templates/excititor-connector/) - **11 – [Authority Service](11_AUTHORITY.md)** - **11 – [Data Schemas](11_DATA_SCHEMAS.md)** - **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** - **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** +- **20 – [CLI AOC Commands Reference](cli/cli-reference.md)** +- **60 – [Policy Engine Overview](policy/overview.md)** +- **61 – [Policy DSL Grammar](policy/dsl.md)** +- **62 – [Policy Lifecycle & Approvals](policy/lifecycle.md)** +- **63 – [Policy Runs & Orchestration](policy/runs.md)** +- **64 – [Policy Engine REST API](api/policy.md)** +- **65 – [Policy CLI Guide](cli/policy.md)** +- **66 – [Policy Editor Workspace](ui/policy-editor.md)** +- **67 – [Policy Observability](observability/policy.md)** +- **68 – [Policy Governance & Least Privilege](security/policy-governance.md)** +- **69 – [Policy Examples](examples/policies/README.md)** +- **70 – [Policy FAQ](faq/policy-faq.md)** +- **71 – [Policy Run DTOs](../src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md)** - **30 – [Fixture Maintenance](dev/fixtures.md)** ### User & operator guides - **14 – [Glossary](14_GLOSSARY_OF_TERMS.md)** - **15 – [UI Guide](15_UI_GUIDE.md)** +- **16 – [Console AOC Dashboard](ui/console.md)** - **17 – [Security Hardening Guide](17_SECURITY_HARDENING_GUIDE.md)** - **18 – [Coding Standards](18_CODING_STANDARDS.md)** - **19 – [Test‑Suite Overview](19_TEST_SUITE_OVERVIEW.md)** @@ -80,9 +97,19 @@ Everything here is open‑source and versioned — when you check out a git ta - **29 – [Concelier CISA ICS Connector Operations](ops/concelier-icscisa-operations.md)** - **30 – [Concelier CERT-Bund Connector Operations](ops/concelier-certbund-operations.md)** - **31 – [Concelier MSRC Connector – AAD Onboarding](ops/concelier-msrc-operations.md)** -- **32 – [Scanner Analyzer Bench Operations](ops/scanner-analyzers-operations.md)** -- **33 – [Scanner Artifact Store Migration](ops/scanner-rustfs-migration.md)** -- **34 – [Zastava Runtime Operations Runbook](ops/zastava-runtime-operations.md)** + - **32 – [Scanner Analyzer Bench Operations](ops/scanner-analyzers-operations.md)** + - **33 – [Scanner Artifact Store Migration](ops/scanner-rustfs-migration.md)** + - **34 – [Zastava Runtime Operations Runbook](ops/zastava-runtime-operations.md)** + - **35 – [Launch Readiness Checklist](ops/launch-readiness.md)** +- **36 – [Launch Cutover Runbook](ops/launch-cutover.md)** +- **37 – [Registry Token Service](ops/registry-token-service.md)** +- **37 – [Deployment Upgrade & Rollback Runbook](ops/deployment-upgrade-runbook.md)** +- **38 – [Policy Schema Export Automation](devops/policy-schema-export.md)** +- **40 – [Observability Guide (AOC)](observability/observability.md)** +- **41 – [Telemetry Collector Deployment](ops/telemetry-collector.md)** +- **42 – [Telemetry Storage Deployment](ops/telemetry-storage.md)** +- **43 – [Authority Scopes & Tenancy](security/authority-scopes.md)** +- **44 – [Container Deployment (AOC)](deploy/containers.md)** ### Legal & licence - **32 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** diff --git a/docs/TASKS.md b/docs/TASKS.md index 9af34349..da1fae4b 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -16,8 +16,9 @@ | PLATFORM-EVENTS-09-401 | DONE (2025-10-21) | Platform Events Guild | DOCS-EVENTS-09-003 | Embed canonical event samples into contract/integration tests and ensure CI validates payloads against published schemas. | Notify models tests now run schema validation against `docs/events/*.json`, event schemas allow optional `attributes`, and docs capture the new validation workflow. | | RUNTIME-GUILD-09-402 | DONE (2025-10-19) | Runtime Guild | SCANNER-POLICY-09-107 | Confirm Scanner WebService surfaces `quietedFindingCount` and progress hints to runtime consumers; document readiness checklist. | Runtime verification run captures enriched payload; checklist/doc updates merged; stakeholders acknowledge availability. | | DOCS-CONCELIER-07-201 | DONE (2025-10-22) | Docs Guild, Concelier WebService | FEEDWEB-DOCS-01-001 | Final editorial review and publish pass for Concelier authority toggle documentation (Quickstart + operator guide). | Review feedback resolved, publish PR merged, release notes updated with documentation pointer. | -| DOCS-RUNTIME-17-004 | TODO | Docs Guild, Runtime Guild | SCANNER-EMIT-17-701, ZASTAVA-OBS-17-005, DEVOPS-REL-17-002 | Document build-id workflows: SBOM exposure, runtime event payloads (`process.buildId`), Scanner `/policy/runtime` response (`buildIds` list), debug-store layout, and operator guidance for symbol retrieval. | Architecture + operator docs updated with build-id sections (Observer, Scanner, CLI), examples show `readelf` output + debuginfod usage, references linked from Offline Kit/Release guides + CLI help. | -| DOCS-OBS-50-001 | TODO | Docs Guild, Observability Guild | TELEMETRY-OBS-50-001 | Publish `/docs/observability/overview.md` introducing scope, imposed rule banner, architecture diagram, and tenant guarantees. | Doc merged with imposed rule banner; diagram committed; cross-links to telemetry stack + evidence locker docs. | +| DOCS-RUNTIME-17-004 | DONE (2025-10-26) | Docs Guild, Runtime Guild | SCANNER-EMIT-17-701, ZASTAVA-OBS-17-005, DEVOPS-REL-17-002 | Document build-id workflows: SBOM exposure, runtime event payloads (`process.buildId`), Scanner `/policy/runtime` response (`buildIds` list), debug-store layout, and operator guidance for symbol retrieval. | Architecture + operator docs updated with build-id sections (Observer, Scanner, CLI), examples show `readelf` output + debuginfod usage, references linked from Offline Kit/Release guides + CLI help. | +| DOCS-OBS-50-001 | BLOCKED (2025-10-26) | Docs Guild, Observability Guild | TELEMETRY-OBS-50-001 | Publish `/docs/observability/overview.md` introducing scope, imposed rule banner, architecture diagram, and tenant guarantees. | Doc merged with imposed rule banner; diagram committed; cross-links to telemetry stack + evidence locker docs. | +> Blocked: waiting on telemetry core deliverable (TELEMETRY-OBS-50-001) to finalise architecture details and diagrams. | DOCS-OBS-50-002 | TODO | Docs Guild, Security Guild | TELEMETRY-OBS-50-002 | Author `/docs/observability/telemetry-standards.md` detailing common fields, scrubbing policy, sampling defaults, and redaction override procedure. | Doc merged; imposed rule banner present; examples validated with telemetry fixtures; security review sign-off captured. | | DOCS-OBS-50-003 | TODO | Docs Guild, Observability Guild | TELEMETRY-OBS-50-001 | Create `/docs/observability/logging.md` covering structured log schema, dos/don'ts, tenant isolation, and copyable examples. | Doc merged with banner; sample logs redacted; lint passes; linked from coding standards. | | DOCS-OBS-50-004 | TODO | Docs Guild, Observability Guild | TELEMETRY-OBS-50-002 | Draft `/docs/observability/tracing.md` explaining context propagation, async linking, CLI header usage, and sampling strategies. | Doc merged; imposed rule banner included; diagrams updated; references to CLI/Console features added. | @@ -32,14 +33,17 @@ | DOCS-CLI-OBS-52-001 | TODO | Docs Guild, DevEx/CLI Guild | CLI-OBS-52-001 | Create `/docs/cli/observability.md` detailing `stella obs` commands, examples, exit codes, imposed rule banner, and scripting tips. | Doc merged; examples tested; banner included; CLI parity matrix updated. | | DOCS-CLI-FORENSICS-53-001 | TODO | Docs Guild, DevEx/CLI Guild | CLI-FORENSICS-54-001 | Publish `/docs/cli/forensics.md` for snapshot/verify/attest commands with sample outputs, imposed rule banner, and offline workflows. | Doc merged; sample bundles verified; banner present; offline notes cross-linked. | | DOCS-RUNBOOK-55-001 | TODO | Docs Guild, Ops Guild | DEVOPS-OBS-55-001, WEB-OBS-55-001 | Author `/docs/runbooks/incidents.md` describing incident mode activation, escalation steps, retention impact, verification checklist, and imposed rule banner. | Doc merged; runbook rehearsed; banner included; linked from alerts. | -| DOCS-AOC-19-001 | TODO | Docs Guild, Concelier Guild | CONCELIER-WEB-AOC-19-001, EXCITITOR-WEB-AOC-19-001 | Author `/docs/ingestion/aggregation-only-contract.md` covering philosophy, invariants, schemas, error codes, migration, observability, and security checklist. | New doc published with compliance checklist; cross-links from existing docs added. | -| DOCS-AOC-19-002 | TODO | Docs Guild, Architecture Guild | DOCS-AOC-19-001 | Update `/docs/architecture/overview.md` to include AOC boundary, raw stores, and sequence diagram (fetch → guard → raw insert → policy evaluation). | Overview doc updated with diagrams/text; lint passes; stakeholders sign off. | -| DOCS-AOC-19-003 | TODO | Docs Guild, Policy Guild | POLICY-AOC-19-003 | Refresh `/docs/architecture/policy-engine.md` clarifying ingestion boundary, raw inputs, and policy-only derived data. | Doc highlights raw-only ingestion contract, updated diagrams merge, compliance checklist added. | -| DOCS-AOC-19-004 | TODO | Docs Guild, UI Guild | UI-AOC-19-001 | Extend `/docs/ui/console.md` with Sources dashboard tiles, violation drill-down workflow, and verification action. | UI doc updated with screenshots/flow descriptions, compliance checklist appended. | -| DOCS-AOC-19-005 | TODO | Docs Guild, CLI Guild | CLI-AOC-19-003 | Update `/docs/cli/cli-reference.md` with `stella sources ingest --dry-run` and `stella aoc verify` usage, exit codes, and offline notes. | CLI reference + quickstart sections updated; examples validated; compliance checklist added. | -| DOCS-AOC-19-006 | TODO | Docs Guild, Observability Guild | CONCELIER-WEB-AOC-19-002, EXCITITOR-WEB-AOC-19-002 | Document new metrics/traces/log keys in `/docs/observability/observability.md`. | Observability doc lists new metrics/traces/log fields; dashboards referenced; compliance checklist appended. | -| DOCS-AOC-19-007 | TODO | Docs Guild, Authority Core | AUTH-AOC-19-001 | Update `/docs/security/authority-scopes.md` with new ingestion scopes and tenancy enforcement notes. | Doc reflects new scopes, sample policies updated, compliance checklist added. | -| DOCS-AOC-19-008 | TODO | Docs Guild, DevOps Guild | DEVOPS-AOC-19-002 | Refresh `/docs/deploy/containers.md` to cover validator enablement, guard env flags, and read-only verify user. | Deploy doc updated; offline kit section mentions validator scripts; compliance checklist appended. | +| DOCS-AOC-19-001 | DONE (2025-10-26) | Docs Guild, Concelier Guild | CONCELIER-WEB-AOC-19-001, EXCITITOR-WEB-AOC-19-001 | Author `/docs/ingestion/aggregation-only-contract.md` covering philosophy, invariants, schemas, error codes, migration, observability, and security checklist. | New doc published with compliance checklist; cross-links from existing docs added. | +| DOCS-AOC-19-002 | DONE (2025-10-26) | Docs Guild, Architecture Guild | DOCS-AOC-19-001 | Update `/docs/architecture/overview.md` to include AOC boundary, raw stores, and sequence diagram (fetch → guard → raw insert → policy evaluation). | Overview doc updated with diagrams/text; lint passes; stakeholders sign off. | +| DOCS-AOC-19-003 | DONE (2025-10-26) | Docs Guild, Policy Guild | POLICY-AOC-19-003 | Refresh `/docs/architecture/policy-engine.md` clarifying ingestion boundary, raw inputs, and policy-only derived data. | Doc highlights raw-only ingestion contract, updated diagrams merge, compliance checklist added. | +| DOCS-AOC-19-004 | DONE (2025-10-26) | Docs Guild, UI Guild | UI-AOC-19-001 | Extend `/docs/ui/console.md` with Sources dashboard tiles, violation drill-down workflow, and verification action. | UI doc updated with screenshots/flow descriptions, compliance checklist appended. | +> DOCS-AOC-19-004: Architecture overview & policy-engine updates landed 2025-10-26; incorporate the new AOC boundary diagrams and metrics references. +| DOCS-AOC-19-005 | DONE (2025-10-26) | Docs Guild, CLI Guild | CLI-AOC-19-003 | Update `/docs/cli/cli-reference.md` with `stella sources ingest --dry-run` and `stella aoc verify` usage, exit codes, and offline notes. | CLI reference + quickstart sections updated; examples validated; compliance checklist added. | +> DOCS-AOC-19-005: New ingestion reference + architecture overview published 2025-10-26; ensure CLI docs link to both and surface AOC exit codes mapping. +| DOCS-AOC-19-006 | DONE (2025-10-26) | Docs Guild, Observability Guild | CONCELIER-WEB-AOC-19-002, EXCITITOR-WEB-AOC-19-002 | Document new metrics/traces/log keys in `/docs/observability/observability.md`. | Observability doc lists new metrics/traces/log fields; dashboards referenced; compliance checklist appended. | +| DOCS-AOC-19-007 | DONE (2025-10-26) | Docs Guild, Authority Core | AUTH-AOC-19-001 | Update `/docs/security/authority-scopes.md` with new ingestion scopes and tenancy enforcement notes. | Doc reflects new scopes, sample policies updated, compliance checklist added. | +| DOCS-AOC-19-008 | DONE (2025-10-26) | Docs Guild, DevOps Guild | DEVOPS-AOC-19-002 | Refresh `/docs/deploy/containers.md` to cover validator enablement, guard env flags, and read-only verify user. | Deploy doc updated; offline kit section mentions validator scripts; compliance checklist appended. | +| DOCS-AOC-19-009 | DONE (2025-10-26) | Docs Guild, Authority Core | AUTH-AOC-19-001 | Update AOC docs/samples to reflect new `advisory:*`, `vex:*`, and `aoc:verify` scopes. | Docs reference new scopes, samples aligned, compliance checklist updated. | ## Air-Gapped Mode (Epic 16) | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | @@ -102,32 +106,23 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DOCS-POLICY-20-001 | TODO | Docs Guild, Policy Guild | POLICY-ENGINE-20-000 | Author `/docs/policy/overview.md` covering concepts, inputs/outputs, determinism, and compliance checklist. | Doc published with diagrams + glossary; lint passes; checklist included. | -| DOCS-POLICY-20-002 | TODO | Docs Guild, Policy Guild | POLICY-ENGINE-20-001 | Write `/docs/policy/dsl.md` with grammar, built-ins, examples, anti-patterns. | DSL doc includes grammar tables, examples, compliance checklist; validated against parser tests. | -| DOCS-POLICY-20-003 | TODO | Docs Guild, Authority Core | AUTH-POLICY-20-001 | Publish `/docs/policy/lifecycle.md` describing draft→approve workflow, roles, audit, compliance list. | Lifecycle doc linked from UI/CLI help; approvals roles documented; checklist appended. | -| DOCS-POLICY-20-004 | TODO | Docs Guild, Scheduler Guild | SCHED-MODELS-20-001 | Create `/docs/policy/runs.md` detailing run modes, incremental mechanics, cursors, replay. | Run doc includes sequence diagrams + compliance checklist; cross-links to scheduler docs. | -| DOCS-POLICY-20-005 | TODO | Docs Guild, BE-Base Platform Guild | WEB-POLICY-20-001 | Draft `/docs/api/policy.md` describing endpoints, schemas, error codes. | API doc validated against OpenAPI; examples included; checklist appended. | -| DOCS-POLICY-20-006 | TODO | Docs Guild, DevEx/CLI Guild | CLI-POLICY-20-002 | Produce `/docs/cli/policy.md` with command usage, exit codes, JSON output contracts. | CLI doc includes examples, exit codes, compliance checklist. | -| DOCS-POLICY-20-007 | TODO | Docs Guild, UI Guild | UI-POLICY-20-001 | Document `/docs/ui/policy-editor.md` covering editor, simulation, diff workflows, approvals. | UI doc includes screenshots/placeholders, accessibility notes, compliance checklist. | -| DOCS-POLICY-20-008 | TODO | Docs Guild, Architecture Guild | POLICY-ENGINE-20-003 | Write `/docs/architecture/policy-engine.md` (new epic content) with sequence diagrams, selection strategy, schema. | Architecture doc merged with diagrams; compliance checklist appended; references updated. | -| DOCS-POLICY-20-009 | TODO | Docs Guild, Observability Guild | POLICY-ENGINE-20-007 | Add `/docs/observability/policy.md` for metrics/traces/logs, sample dashboards. | Observability doc includes metrics tables, dashboard screenshots, checklist. | -| DOCS-POLICY-20-010 | TODO | Docs Guild, Security Guild | AUTH-POLICY-20-002 | Publish `/docs/security/policy-governance.md` covering scopes, approvals, tenancy, least privilege. | Security doc merged; compliance checklist appended; reviewed by Security Guild. | -| DOCS-POLICY-20-011 | TODO | Docs Guild, Policy Guild | POLICY-ENGINE-20-001 | Populate `/docs/examples/policies/` with baseline/serverless/internal-only samples and commentary. | Example policies committed with explanations; lint passes; compliance checklist per file. | -| DOCS-POLICY-20-012 | TODO | Docs Guild, Support Guild | WEB-POLICY-20-003 | Draft `/docs/faq/policy-faq.md` addressing common pitfalls, VEX conflicts, determinism issues. | FAQ published with Q/A entries, cross-links, compliance checklist. | +| DOCS-POLICY-20-001 | DONE (2025-10-26) | Docs Guild, Policy Guild | POLICY-ENGINE-20-000 | Author `/docs/policy/overview.md` covering concepts, inputs/outputs, determinism, and compliance checklist. | Doc published with diagrams + glossary; lint passes; checklist included. | +| DOCS-POLICY-20-002 | DONE (2025-10-26) | Docs Guild, Policy Guild | POLICY-ENGINE-20-001 | Write `/docs/policy/dsl.md` with grammar, built-ins, examples, anti-patterns. | DSL doc includes grammar tables, examples, compliance checklist; validated against parser tests. | +| DOCS-POLICY-20-003 | DONE (2025-10-26) | Docs Guild, Authority Core | AUTH-POLICY-20-001 | Publish `/docs/policy/lifecycle.md` describing draft→approve workflow, roles, audit, compliance list. | Lifecycle doc linked from UI/CLI help; approvals roles documented; checklist appended. | +| DOCS-POLICY-20-004 | DONE (2025-10-26) | Docs Guild, Scheduler Guild | SCHED-MODELS-20-001 | Create `/docs/policy/runs.md` detailing run modes, incremental mechanics, cursors, replay. | Run doc includes sequence diagrams + compliance checklist; cross-links to scheduler docs. | +| DOCS-POLICY-20-005 | DONE (2025-10-26) | Docs Guild, BE-Base Platform Guild | WEB-POLICY-20-001 | Draft `/docs/api/policy.md` describing endpoints, schemas, error codes. | API doc validated against OpenAPI; examples included; checklist appended. | +| DOCS-POLICY-20-006 | DONE (2025-10-26) | Docs Guild, DevEx/CLI Guild | CLI-POLICY-20-002 | Produce `/docs/cli/policy.md` with command usage, exit codes, JSON output contracts. | CLI doc includes examples, exit codes, compliance checklist. | +| DOCS-POLICY-20-007 | DONE (2025-10-26) | Docs Guild, UI Guild | UI-POLICY-20-001 | Document `/docs/ui/policy-editor.md` covering editor, simulation, diff workflows, approvals. | UI doc includes screenshots/placeholders, accessibility notes, compliance checklist. | +| DOCS-POLICY-20-008 | DONE (2025-10-26) | Docs Guild, Architecture Guild | POLICY-ENGINE-20-003 | Write `/docs/architecture/policy-engine.md` (new epic content) with sequence diagrams, selection strategy, schema. | Architecture doc merged with diagrams; compliance checklist appended; references updated. | +| DOCS-POLICY-20-009 | DONE (2025-10-26) | Docs Guild, Observability Guild | POLICY-ENGINE-20-007 | Add `/docs/observability/policy.md` for metrics/traces/logs, sample dashboards. | Observability doc includes metrics tables, dashboard screenshots, checklist. | +| DOCS-POLICY-20-010 | DONE (2025-10-26) | Docs Guild, Security Guild | AUTH-POLICY-20-002 | Publish `/docs/security/policy-governance.md` covering scopes, approvals, tenancy, least privilege. | Security doc merged; compliance checklist appended; reviewed by Security Guild. | +| DOCS-POLICY-20-011 | DONE (2025-10-26) | Docs Guild, Policy Guild | POLICY-ENGINE-20-001 | Populate `/docs/examples/policies/` with baseline/serverless/internal-only samples and commentary. | Example policies committed with explanations; lint passes; compliance checklist per file. | +| DOCS-POLICY-20-012 | DONE (2025-10-26) | Docs Guild, Support Guild | WEB-POLICY-20-003 | Draft `/docs/faq/policy-faq.md` addressing common pitfalls, VEX conflicts, determinism issues. | FAQ published with Q/A entries, cross-links, compliance checklist. | ## Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DOCS-GRAPH-21-001 | TODO | Docs Guild, Cartographer Guild | CARTO-GRAPH-21-001..006 | Author `/docs/graph/overview.md` covering concepts, snapshot lifecycle, overlays, and compliance checklist. | Doc merged with diagrams; lint passes; checklist appended. | -| DOCS-GRAPH-21-002 | TODO | Docs Guild | CARTO-GRAPH-21-001 | Write `/docs/graph/schema.md` describing node/edge/overlay schemas, indexes, sharding strategy, and sample docs. | Schema doc validated against fixtures; compliance checklist included. | -| DOCS-GRAPH-21-003 | TODO | Docs Guild, BE-Base Platform Guild | WEB-GRAPH-21-001..004 | Produce `/docs/graph/api.md` with endpoint specs, parameters, pagination, errors, and curl examples. | API doc aligns with OpenAPI; examples verified; checklist appended. | -| DOCS-GRAPH-21-004 | TODO | Docs Guild, UI Guild | UI-GRAPH-21-001..006 | Document `/docs/ui/graph-explorer.md` (screens, filters, paths, diff, accessibility). | UI doc published with screenshots/placeholders; accessibility checklist satisfied. | -| DOCS-GRAPH-21-005 | TODO | Docs Guild, DevEx/CLI Guild | CLI-GRAPH-21-001..003 | Create `/docs/cli/graph.md` detailing CLI commands, exit codes, JSON schemas. | CLI doc merged; examples validated; checklist appended. | -| DOCS-GRAPH-21-006 | TODO | Docs Guild, Architecture Guild | CARTO-GRAPH-21-002..007 | Draft `/docs/architecture/cartographer.md` covering build pipeline, layout tiling, overlay patching, sequence diagrams. | Architecture doc merged with diagrams; compliance checklist included. | -| DOCS-GRAPH-21-007 | TODO | Docs Guild, Observability Guild | CARTO-GRAPH-21-008, DEVOPS-GRAPH-21-001 | Publish `/docs/observability/graph.md` (metrics/traces/logs, dashboards, alerts). | Observability doc live; dashboards linked; checklist appended. | -| DOCS-GRAPH-21-008 | TODO | Docs Guild, Security Guild | AUTH-GRAPH-21-001..003 | Write `/docs/security/graph-access.md` describing RBAC, tenancy, scopes, service identities. | Security doc merged; reviewer checklist completed. | -| DOCS-GRAPH-21-009 | TODO | Docs Guild, Cartographer Guild | CARTO-GRAPH-21-006 | Document `/docs/examples/graph/` sample SBOMs, screenshots, exports with reviewer checklist. | Example docs + assets committed; lint passes; checklist appended. | ## Link-Not-Merge v1 @@ -143,17 +138,17 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DOCS-CONSOLE-23-001 | TODO | Docs Guild, Console Guild | CONSOLE-CORE-23-004 | Publish `/docs/ui/console-overview.md` covering IA, tenant model, global filters, and AOC alignment with compliance checklist. | Doc merged with diagrams + overview tables; checklist appended; Console Guild sign-off. | -| DOCS-CONSOLE-23-002 | TODO | Docs Guild, Console Guild | DOCS-CONSOLE-23-001 | Author `/docs/ui/navigation.md` detailing routes, breadcrumbs, keyboard shortcuts, deep links, and tenant context switching. | Navigation doc merged with shortcut tables and screenshots; accessibility checklist satisfied. | -| DOCS-CONSOLE-23-003 | TODO | Docs Guild, SBOM Service Guild, Console Guild | SBOM-CONSOLE-23-001, CONSOLE-FEAT-23-102 | Document `/docs/ui/sbom-explorer.md` (catalog, detail, graph overlays, exports) including compliance checklist and performance tips. | Doc merged with annotated screenshots, export instructions, and overlay examples; checklist appended. | -| DOCS-CONSOLE-23-004 | TODO | Docs Guild, Concelier Guild, Excititor Guild | CONCELIER-CONSOLE-23-001, EXCITITOR-CONSOLE-23-001 | Produce `/docs/ui/advisories-and-vex.md` explaining aggregation-not-merge, conflict indicators, raw viewers, and provenance banners. | Doc merged; raw JSON examples included; compliance checklist complete. | -| DOCS-CONSOLE-23-005 | TODO | Docs Guild, Policy Guild | POLICY-CONSOLE-23-001, CONSOLE-FEAT-23-104 | Write `/docs/ui/findings.md` describing filters, saved views, explain drawer, exports, and CLI parity callouts. | Doc merged with filter matrix + explain walkthrough; checklist appended. | -| DOCS-CONSOLE-23-006 | TODO | Docs Guild, Policy Guild, Product Ops | POLICY-CONSOLE-23-002, CONSOLE-FEAT-23-105 | Publish `/docs/ui/policies.md` with editor, simulation, approvals, compliance checklist, and RBAC mapping. | Doc merged; Monaco screenshots + simulation diff examples included; approval flow described; checklist appended. | -| DOCS-CONSOLE-23-007 | TODO | Docs Guild, Scheduler Guild | SCHED-CONSOLE-23-001, CONSOLE-FEAT-23-106 | Document `/docs/ui/runs.md` covering queues, live progress, diffs, retries, evidence downloads, and troubleshooting. | Doc merged with SSE troubleshooting, metrics references, compliance checklist. | -| DOCS-CONSOLE-23-008 | TODO | Docs Guild, Authority Guild | AUTH-CONSOLE-23-002, CONSOLE-FEAT-23-108 | Draft `/docs/ui/admin.md` describing users/roles, tenants, tokens, integrations, fresh-auth prompts, and RBAC mapping. | Doc merged with tables for scopes vs roles, screenshots, compliance checklist. | -| DOCS-CONSOLE-23-009 | TODO | Docs Guild, DevOps Guild | DOWNLOADS-CONSOLE-23-001, CONSOLE-FEAT-23-109 | Publish `/docs/ui/downloads.md` listing product images, commands, offline instructions, parity with CLI, and compliance checklist. | Doc merged; manifest sample included; copy-to-clipboard guidance documented; checklist complete. | -| DOCS-CONSOLE-23-010 | TODO | Docs Guild, Deployment Guild, Console Guild | DEVOPS-CONSOLE-23-002, CONSOLE-REL-23-301 | Write `/docs/deploy/console.md` (Helm, ingress, TLS, CSP, env vars, health checks) with compliance checklist. | Deploy doc merged; templates validated; CSP guidance included; checklist appended. | -| DOCS-CONSOLE-23-011 | TODO | Docs Guild, Deployment Guild | DOCS-CONSOLE-23-010 | Update `/docs/install/docker.md` to cover Console image, Compose/Helm usage, offline tarballs, parity with CLI. | Doc updated with new sections; commands validated; compliance checklist appended. | +| DOCS-CONSOLE-23-001 | DONE (2025-10-26) | Docs Guild, Console Guild | CONSOLE-CORE-23-004 | Publish `/docs/ui/console-overview.md` covering IA, tenant model, global filters, and AOC alignment with compliance checklist. | Doc merged with diagrams + overview tables; checklist appended; Console Guild sign-off. | +| DOCS-CONSOLE-23-002 | DONE (2025-10-26) | Docs Guild, Console Guild | DOCS-CONSOLE-23-001 | Author `/docs/ui/navigation.md` detailing routes, breadcrumbs, keyboard shortcuts, deep links, and tenant context switching. | Navigation doc merged with shortcut tables and screenshots; accessibility checklist satisfied. | +| DOCS-CONSOLE-23-003 | DONE (2025-10-26) | Docs Guild, SBOM Service Guild, Console Guild | SBOM-CONSOLE-23-001, CONSOLE-FEAT-23-102 | Document `/docs/ui/sbom-explorer.md` (catalog, detail, graph overlays, exports) including compliance checklist and performance tips. | Doc merged with annotated screenshots, export instructions, and overlay examples; checklist appended. | +| DOCS-CONSOLE-23-004 | DONE (2025-10-26) | Docs Guild, Concelier Guild, Excititor Guild | CONCELIER-CONSOLE-23-001, EXCITITOR-CONSOLE-23-001 | Produce `/docs/ui/advisories-and-vex.md` explaining aggregation-not-merge, conflict indicators, raw viewers, and provenance banners. | Doc merged; raw JSON examples included; compliance checklist complete. | +| DOCS-CONSOLE-23-005 | DONE (2025-10-26) | Docs Guild, Policy Guild | POLICY-CONSOLE-23-001, CONSOLE-FEAT-23-104 | Write `/docs/ui/findings.md` describing filters, saved views, explain drawer, exports, and CLI parity callouts. | Doc merged with filter matrix + explain walkthrough; checklist appended. | +| DOCS-CONSOLE-23-006 | DONE (2025-10-26) | Docs Guild, Policy Guild, Product Ops | POLICY-CONSOLE-23-002, CONSOLE-FEAT-23-105 | Publish `/docs/ui/policies.md` with editor, simulation, approvals, compliance checklist, and RBAC mapping. | Doc merged; Monaco screenshots + simulation diff examples included; approval flow described; checklist appended. | +| DOCS-CONSOLE-23-007 | DONE (2025-10-26) | Docs Guild, Scheduler Guild | SCHED-CONSOLE-23-001, CONSOLE-FEAT-23-106 | Document `/docs/ui/runs.md` covering queues, live progress, diffs, retries, evidence downloads, and troubleshooting. | Doc merged with SSE troubleshooting, metrics references, compliance checklist. | +| DOCS-CONSOLE-23-008 | DONE (2025-10-26) | Docs Guild, Authority Guild | AUTH-CONSOLE-23-002, CONSOLE-FEAT-23-108 | Draft `/docs/ui/admin.md` describing users/roles, tenants, tokens, integrations, fresh-auth prompts, and RBAC mapping. | Doc merged with tables for scopes vs roles, screenshots, compliance checklist. | +| DOCS-CONSOLE-23-009 | DONE (2025-10-27) | Docs Guild, DevOps Guild | DOWNLOADS-CONSOLE-23-001, CONSOLE-FEAT-23-109 | Publish `/docs/ui/downloads.md` listing product images, commands, offline instructions, parity with CLI, and compliance checklist. | Doc merged; manifest sample included; copy-to-clipboard guidance documented; checklist complete. | +| DOCS-CONSOLE-23-010 | DONE (2025-10-27) | Docs Guild, Deployment Guild, Console Guild | DEVOPS-CONSOLE-23-002, CONSOLE-REL-23-301 | Write `/docs/deploy/console.md` (Helm, ingress, TLS, CSP, env vars, health checks) with compliance checklist. | Deploy doc merged; templates validated; CSP guidance included; checklist appended. | +| DOCS-CONSOLE-23-011 | DOING (2025-10-27) | Docs Guild, Deployment Guild | DOCS-CONSOLE-23-010 | Update `/docs/install/docker.md` to cover Console image, Compose/Helm usage, offline tarballs, parity with CLI. | Doc updated with new sections; commands validated; compliance checklist appended. | | DOCS-CONSOLE-23-012 | TODO | Docs Guild, Security Guild | AUTH-CONSOLE-23-003, WEB-CONSOLE-23-002 | Publish `/docs/security/console-security.md` detailing OIDC flows, scopes, CSP, fresh-auth, evidence handling, and compliance checklist. | Security doc merged; threat model notes included; checklist appended. | | DOCS-CONSOLE-23-013 | TODO | Docs Guild, Observability Guild | TELEMETRY-CONSOLE-23-001, CONSOLE-QA-23-403 | Write `/docs/observability/ui-telemetry.md` cataloguing metrics/logs/traces, dashboards, alerts, and feature flags. | Doc merged with instrumentation tables, dashboard screenshots, checklist appended. | | DOCS-CONSOLE-23-014 | TODO | Docs Guild, Console Guild, CLI Guild | CONSOLE-DOC-23-502 | Maintain `/docs/cli-vs-ui-parity.md` matrix and integrate CI check guidance. | Matrix published with parity status, CI workflow documented, compliance checklist appended. | diff --git a/docs/aoc/aoc-guardrails.md b/docs/aoc/aoc-guardrails.md index 568d969a..686881f7 100644 --- a/docs/aoc/aoc-guardrails.md +++ b/docs/aoc/aoc-guardrails.md @@ -9,3 +9,5 @@ The Aggregation-Only Contract keeps ingestion services deterministic and policy- 5. **Guardrails everywhere.** Roslyn analyzers, schema validators, and CI smoke tests should fail builds that attempt forbidden writes. For detailed roles and ownership boundaries, see `AGENTS.md` at the repo root and the module-specific `ARCHITECTURE_*.md` dossiers. + +Need the full contract? Read the [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) for schemas, error codes, and migration guidance. diff --git a/docs/api/policy.md b/docs/api/policy.md new file mode 100644 index 00000000..31f14c53 --- /dev/null +++ b/docs/api/policy.md @@ -0,0 +1,402 @@ +# Policy Engine REST API + +> **Audience:** Backend integrators, platform operators, and CI engineers invoking Policy Engine services programmatically. +> **Base URL:** `/api/policy/*` (internal gateway route) – requires OAuth 2.0 bearer token issued by Authority with scopes listed below. + +This document is the canonical reference for the Policy Engine REST surface described in Epic 2 (Policy Engine v2). Use it alongside the [DSL](../policy/dsl.md), [Lifecycle](../policy/lifecycle.md), and [Runs](../policy/runs.md) guides for end-to-end implementations. + +--- + +## 1 · Authentication & Headers + +- **Auth:** Bearer tokens (`Authorization: Bearer `) with the following scopes as applicable: + - `policy:read`, `policy:write`, `policy:submit`, `policy:approve`, `policy:run`, `policy:activate`, `policy:archive`, `policy:simulate`, `policy:runs` + - `findings:read` (for effective findings APIs) + - `effective:write` (service identity only; not exposed to clients) +- **Service identity:** Authority marks the Policy Engine client with `properties.serviceIdentity: policy-engine`. Tokens missing this marker cannot obtain `effective:write`. +- **Tenant:** Supply tenant context via `X-Stella-Tenant`. Tokens without multi-tenant claims default to `default`. +- **Idempotency:** For mutating endpoints, include `Idempotency-Key` (UUID). Retries with same key return original result. +- **Content type:** All request/response bodies are `application/json; charset=utf-8` unless otherwise noted. + +--- + +## 2 · Error Model + +All errors use HTTP semantics plus a structured payload: + +```json +{ + "code": "ERR_POL_001", + "message": "Policy syntax error", + "details": [ + {"path": "rules[0].when", "error": "Unknown function foo()"} + ], + "traceId": "01HDV1C4E9Z4T5G6H7J8", + "timestamp": "2025-10-26T14:07:03Z" +} +``` + +| Code | Meaning | Notes | +|------|---------|-------| +| `ERR_POL_001` | Policy syntax/compile error | Returned by `compile`, `submit`, `simulate`, `run` when DSL invalid. | +| `ERR_POL_002` | Policy not approved | Attempted to run or activate unapproved version. | +| `ERR_POL_003` | Missing inputs | Downstream service unavailable (Concelier/Excititor/SBOM). | +| `ERR_POL_004` | Determinism violation | Illegal API usage (wall-clock, RNG). Triggers incident mode. | +| `ERR_POL_005` | Unauthorized materialisation | Identity lacks `effective:write`. | +| `ERR_POL_006` | Run canceled or timed out | Includes cancellation metadata. | + +--- + +## 3 · Policy Management + +### 3.1 Create Draft + +``` +POST /api/policy/policies +Scopes: policy:write +``` + +**Request** + +```json +{ + "policyId": "P-7", + "name": "Default Org Policy", + "description": "Baseline severity + VEX precedence", + "dsl": { + "syntax": "stella-dsl@1", + "source": "policy \"Default Org Policy\" syntax \"stella-dsl@1\" { ... }" + }, + "tags": ["baseline","vex"] +} +``` + +**Response 201** + +```json +{ + "policyId": "P-7", + "version": 1, + "status": "draft", + "digest": "sha256:7e1d…", + "createdBy": "user:ali", + "createdAt": "2025-10-26T13:40:00Z" +} +``` + +### 3.2 List Policies + +``` +GET /api/policy/policies?status=approved&tenant=default&page=1&pageSize=25 +Scopes: policy:read +``` + +Returns paginated list with `X-Total-Count` header. + +### 3.3 Fetch Version + +``` +GET /api/policy/policies/{policyId}/versions/{version} +Scopes: policy:read +``` + +Returns full DSL, metadata, provenance, simulation artefact references. + +### 3.4 Update Draft Version + +``` +PUT /api/policy/policies/{policyId}/versions/{version} +Scopes: policy:write +``` + +Body identical to create. Only permitted while `status=draft`. + +--- + +## 4 · Lifecycle Transitions + +### 4.1 Submit for Review + +``` +POST /api/policy/policies/{policyId}/versions/{version}:submit +Scopes: policy:submit +``` + +**Request** + +```json +{ + "reviewers": ["user:kay","group:sec-reviewers"], + "notes": "Simulated on golden SBOM set (diff attached)", + "simulationArtifacts": [ + "blob://policy/P-7/v3/simulations/2025-10-26.json" + ] +} +``` + +**Response 202** – submission recorded. `Location` header points to review resource. + +### 4.2 Review Feedback + +``` +POST /api/policy/policies/{policyId}/versions/{version}/reviews +Scopes: policy:review +``` + +**Request** + +```json +{ + "decision": "approve", // approve | request_changes | comment + "note": "Looks good, ensure incident playbook covers reachability data.", + "blocking": false +} +``` + +### 4.3 Approve + +``` +POST /api/policy/policies/{policyId}/versions/{version}:approve +Scopes: policy:approve +``` + +Body requires approval note and confirmation of compliance gates: + +```json +{ + "note": "All simulations and determinism checks passed.", + "acknowledgeDeterminism": true, + "acknowledgeSimulation": true +} +``` + +### 4.4 Activate + +``` +POST /api/policy/policies/{policyId}/versions/{version}:activate +Scopes: policy:activate, policy:run +``` + +Marks version as active for tenant; triggers optional immediate full run (`"runNow": true`). + +### 4.5 Archive + +``` +POST /api/policy/policies/{policyId}/versions/{version}:archive +Scopes: policy:archive +``` + +Request includes `reason` and optional `incidentId`. + +--- + +## 5 · Compilation & Validation + +### 5.1 Compile + +``` +POST /api/policy/policies/{policyId}/versions/{version}:compile +Scopes: policy:write +``` + +**Response 200** + +```json +{ + "digest": "sha256:7e1d…", + "warnings": [], + "rules": { + "count": 24, + "actions": { + "block": 5, + "warn": 4, + "ignore": 3, + "requireVex": 2 + } + } +} +``` + +### 5.2 Lint / Simulate Quick Check + +``` +POST /api/policy/policies/{policyId}/lint +Scopes: policy:write +``` + +Slim wrapper used by CLI; returns 204 on success or `ERR_POL_001` payload. + +--- + +## 6 · Run & Simulation APIs + +> Schema reference: canonical policy run request/status/diff payloads ship with the Scheduler Models guide (`src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md`) and JSON fixtures under `samples/api/scheduler/policy-*.json`. + +### 6.1 Trigger Run + +``` +POST /api/policy/policies/{policyId}/runs +Scopes: policy:run +``` + +**Request** + +```json +{ + "mode": "incremental", // full | incremental + "runId": "run:P-7:2025-10-26:auto", // optional idempotency key + "sbomSet": ["sbom:S-42","sbom:S-318"], + "env": {"exposure": "internet"}, + "priority": "normal" // normal | high | emergency +} +``` + +**Response 202** + +```json +{ + "runId": "run:P-7:2025-10-26:auto", + "status": "queued", + "queuedAt": "2025-10-26T14:05:00Z" +} +``` + +### 6.2 Get Run Status + +``` +GET /api/policy/policies/{policyId}/runs/{runId} +Scopes: policy:runs +``` + +Includes status, stats, determinism hash, failure diagnostics. + +### 6.3 List Runs + +``` +GET /api/policy/policies/{policyId}/runs?mode=incremental&status=failed&page=1&pageSize=20 +Scopes: policy:runs +``` + +Supports filtering by `mode`, `status`, `from`/`to` timestamps, `tenant`. + +### 6.4 Simulate + +``` +POST /api/policy/policies/{policyId}/simulate +Scopes: policy:simulate +``` + +**Request** + +```json +{ + "baseVersion": 3, + "candidateVersion": 4, + "sbomSet": ["sbom:S-42","sbom:S-318"], + "env": {"sealed": false}, + "explain": true +} +``` + +**Response 200** + +```json +{ + "diff": { + "added": 12, + "removed": 8, + "unchanged": 657, + "bySeverity": { + "Critical": {"up": 1, "down": 0}, + "High": {"up": 3, "down": 4} + } + }, + "explainUri": "blob://policy/P-7/simulations/2025-10-26-4-vs-3.json" +} +``` + +### 6.5 Replay Run + +``` +POST /api/policy/policies/{policyId}/runs/{runId}:replay +Scopes: policy:runs, policy:simulate +``` + +Produces sealed bundle for determinism verification; returns location of bundle. + +--- + +## 7 · Effective Findings APIs + +### 7.1 List Findings + +``` +GET /api/policy/findings/{policyId}?sbomId=S-42&status=affected&severity=High,Critical&page=1&pageSize=100 +Scopes: findings:read +``` + +Response includes cursor-based pagination: + +```json +{ + "items": [ + { + "findingId": "P-7:S-42:pkg:npm/lodash@4.17.21:CVE-2021-23337", + "status": "affected", + "severity": {"normalized": "High", "score": 7.5}, + "sbomId": "sbom:S-42", + "advisoryIds": ["CVE-2021-23337"], + "vex": {"winningStatementId": "VendorX-123"}, + "policyVersion": 4, + "updatedAt": "2025-10-26T14:06:01Z" + } + ], + "nextCursor": "eyJwYWdlIjoxfQ==" +} +``` + +### 7.2 Fetch Explain Trace + +``` +GET /api/policy/findings/{policyId}/{findingId}/explain?mode=verbose +Scopes: findings:read +``` + +Returns rule hit sequence: + +```json +{ + "findingId": "P-7:S-42:pkg:npm/lodash@4.17.21:CVE-2021-23337", + "policyVersion": 4, + "steps": [ + {"rule": "vex_precedence", "status": "not_affected", "inputs": {"statementId": "VendorX-123"}}, + {"rule": "severity_baseline", "severity": {"normalized": "Low", "score": 3.4}} + ], + "sealedHints": [{"message": "Using cached EPSS percentile from bundle 2025-10-20"}] +} +``` + +--- + +## 8 · Events & Webhooks + +- `policy.run.completed` – emitted with `runId`, `policyId`, `mode`, `stats`, `determinismHash`. +- `policy.run.failed` – includes error code, retry count, guidance. +- `policy.lifecycle.*` – mirrored from lifecycle APIs (see [Lifecycle guide](../policy/lifecycle.md)). +- Webhook registration occurs via `/api/policy/webhooks` (future work, reserved). For now, integrate with Notifier streams documented in `/docs/notifications/*`. + +--- + +## 9 · Compliance Checklist + +- [ ] **Scopes enforced:** Endpoint access requires correct Authority scope mapping (see `/src/StellaOps.Authority/TASKS.md`). +- [ ] **Schemas current:** JSON examples align with Scheduler Models (`SCHED-MODELS-20-001`) and Policy Engine DTOs; update when contracts change. +- [ ] **Error codes mapped:** `ERR_POL_*` table reflects implementation and CI tests cover edge cases. +- [ ] **Pagination documented:** List endpoints specify page/size and cursor semantics; responses include `X-Total-Count` or `nextCursor`. +- [ ] **Idempotency described:** Mutating endpoints mandate `Idempotency-Key`. +- [ ] **Offline parity noted:** Simulate/run endpoints explain `--sealed` behaviour and bundle generation. +- [ ] **Cross-links added:** References to lifecycle, runs, DSL, and observability docs verified. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md new file mode 100644 index 00000000..d4bad751 --- /dev/null +++ b/docs/architecture/overview.md @@ -0,0 +1,168 @@ +# StellaOps Architecture Overview (Sprint 19) + +> **Ownership:** Architecture Guild • Docs Guild +> **Audience:** Service owners, platform engineers, solution architects +> **Related:** [High-Level Architecture](../07_HIGH_LEVEL_ARCHITECTURE.md), [Concelier Architecture](../ARCHITECTURE_CONCELIER.md), [Policy Engine Architecture](policy-engine.md), [Aggregation-Only Contract](../ingestion/aggregation-only-contract.md) + +This dossier summarises the end-to-end runtime topology after the Aggregation-Only Contract (AOC) rollout. It highlights where raw facts live, how ingest services enforce guardrails, and how downstream components consume those facts to derive policy decisions and user-facing experiences. + +--- + +## 1 · System landscape + +```mermaid +graph TD + subgraph Edge["Clients & Automation"] + CLI[stella CLI] + UI[Console SPA] + APIClients[CI / API Clients] + end + Gateway[API Gateway
(JWT + DPoP scopes)] + subgraph Scanner["Fact Collection"] + ScannerWeb[Scanner.WebService] + ScannerWorkers[Scanner.Workers] + Agent[Agent Runtime] + end + subgraph Ingestion["Aggregation-Only Ingestion (AOC)"] + Concelier[Concelier.WebService] + Excititor[Excititor.WebService] + RawStore[(MongoDB
advisory_raw / vex_raw)] + end + subgraph Derivation["Policy & Overlay"] + Policy[Policy Engine] + Scheduler[Scheduler Services] + Notify[Notifier] + end + subgraph Experience["UX & Export"] + UIService[Console Backend] + Exporters[Export / Offline Kit] + end + Observability[Telemetry Stack] + + CLI --> Gateway + UI --> Gateway + APIClients --> Gateway + Gateway --> ScannerWeb + ScannerWeb --> ScannerWorkers + ScannerWorkers --> Concelier + ScannerWorkers --> Excititor + Concelier --> RawStore + Excititor --> RawStore + RawStore --> Policy + Policy --> Scheduler + Policy --> Notify + Policy --> UIService + Scheduler --> UIService + UIService --> Exporters + Exporters --> CLI + Exporters --> Offline[Offline Kit] + Observability -.-> ScannerWeb + Observability -.-> Concelier + Observability -.-> Excititor + Observability -.-> Policy + Observability -.-> Scheduler + Observability -.-> Notify +``` + +Key boundaries: + +- **AOC border.** Everything inside the Ingestion subgraph writes only immutable raw facts plus link hints. Derived severity, consensus, and risk remain outside the border. +- **Policy-only derivation.** Policy Engine materialises `effective_finding_*` collections and emits overlays; other services consume but never mutate them. +- **Tenant enforcement.** Authority-issued DPoP scopes flow through Gateway to every service; raw stores and overlays include `tenant` strictly. + +--- + +## 2 · Aggregation-Only Contract focus + +### 2.1 Responsibilities at the boundary + +| Area | Services | Responsibilities under AOC | Forbidden under AOC | +|------|----------|-----------------------------|---------------------| +| **Ingestion (Concelier / Excititor)** | `StellaOps.Concelier.WebService`, `StellaOps.Excititor.WebService` | Fetch upstream advisories/VEX, verify signatures, compute linksets, append immutable documents to `advisory_raw` / `vex_raw`, emit observability signals, expose raw read APIs. | Computing severity, consensus, suppressions, or policy hints; merging upstream sources into a single derived record; mutating existing documents. | +| **Policy & Overlay** | `StellaOps.Policy.Engine`, Scheduler | Join SBOM inventory with raw advisories/VEX, evaluate policies, issue `effective_finding_*` overlays, drive remediation workflows. | Writing to raw collections; bypassing guard scopes; running without recorded provenance. | +| **Experience layers** | Console, CLI, Exporters | Surface raw facts + policy overlays; run `stella aoc verify`; render AOC dashboards and reports. | Accepting ingestion payloads that lack provenance or violate guard results. | + +### 2.2 Raw stores + +| Collection | Purpose | Key fields | Notes | +|------------|---------|------------|-------| +| `advisory_raw` | Immutable vendor/ecosystem advisory documents. | `_id`, `tenant`, `source.*`, `upstream.*`, `content.raw`, `linkset`, `supersedes`. | Idempotent by `(source.vendor, upstream.upstream_id, upstream.content_hash)`. | +| `vex_raw` | Immutable vendor VEX statements. | Mirrors `advisory_raw`; `identifiers.statements` summarises affected components. | Maintains supersedes chain identical to advisory flow. | +| Change streams (`advisory_raw_stream`, `vex_raw_stream`) | Feed Policy Engine and Scheduler. | `operationType`, `documentKey`, `fullDocument`, `tenant`, `traceId`. | Scope filtered per tenant before delivery. | + +### 2.3 Guarded ingestion sequence + +```mermaid +sequenceDiagram + participant Upstream as Upstream Source + participant Connector as Concelier/Excititor Connector + participant Guard as AOCWriteGuard + participant Mongo as MongoDB (advisory_raw / vex_raw) + participant Stream as Change Stream + participant Policy as Policy Engine + + Upstream-->>Connector: CSAF / OSV / VEX document + Connector->>Connector: Normalize transport, compute content_hash + Connector->>Guard: Candidate raw doc (source + upstream + content + linkset) + Guard-->>Connector: ERR_AOC_00x on violation + Guard->>Mongo: Append immutable document (with tenant & supersedes) + Mongo-->>Stream: Change event (tenant scoped) + Stream->>Policy: Raw delta payload + Policy->>Policy: Evaluate policies, compute effective findings +``` + +--- + +### 2.4 Authority scopes & tenancy + +| Scope | Holder | Purpose | Notes | +|-------|--------|---------|-------| +| `advisory:write` / `vex:write` | Concelier / Excititor collectors | Append raw documents through ingestion endpoints. | Paired with tenant claims; requests without tenant are rejected. | +| `advisory:verify` / `vex:verify` | DevOps verify identity, CLI | Run `stella aoc verify` or call `/aoc/verify`. | Read-only; cannot mutate raw docs. | +| `effective:write` | Policy Engine | Materialise `effective_finding_*` overlays. | Only Policy Engine identity may hold; ingestion contexts receive `ERR_AOC_006` if they attempt. | +| `effective:read` | Console, CLI, exports | Consume derived findings. | Enforced by Gateway and downstream services. | + +--- + +## 3 · Data & control flow highlights + +1. **Ingestion:** Concelier / Excititor connectors fetch upstream documents, compute linksets, and hand payloads to `AOCWriteGuard`. Guards validate schema, provenance, forbidden fields, supersedes pointers, and append-only rules before writing to Mongo. +2. **Verification:** `stella aoc verify` (CLI/CI) and `/aoc/verify` endpoints replay guard checks against stored documents, mapping `ERR_AOC_00x` codes to exit codes for automation. +3. **Policy evaluation:** Mongo change streams deliver tenant-scoped raw deltas. Policy Engine joins SBOM inventory (via BOM Index), executes deterministic policies, writes overlays, and emits events to Scheduler/Notify. +4. **Experience surfaces:** Console renders an AOC dashboard showing ingestion latency, guard violations, and supersedes depth. CLI exposes raw-document fetch helpers for auditing. Offline Kit bundles raw collections alongside guard configs to keep air-gapped installs verifiable. +5. **Observability:** All services emit `ingestion_write_total`, `aoc_violation_total{code}`, `ingestion_latency_seconds`, and trace spans `ingest.fetch`, `ingest.transform`, `ingest.write`, `aoc.guard`. Logs correlate via `traceId`, `tenant`, `source.vendor`, and `content_hash`. + +--- + +## 4 · Offline & disaster readiness + +- **Offline Kit:** Packages raw Mongo snapshots (`advisory_raw`, `vex_raw`) plus guard configuration and CLI verifier binaries so air-gapped sites can re-run AOC checks before promotion. +- **Recovery:** Supersedes chains allow rollback to prior revisions without mutating documents. Disaster exercises must rehearse restoring from snapshot, replaying change streams into Policy Engine, and re-validating guard compliance. +- **Migration:** Legacy normalised fields are moved to temporary views during cutover; ingestion runtime removes writes once guard-enforced path is live (see [Migration playbook](../ingestion/aggregation-only-contract.md#8-migration-playbook)). + +--- + +## 5 · References + +- [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) +- [Concelier architecture](../ARCHITECTURE_CONCELIER.md) +- [Excititor architecture](../ARCHITECTURE_EXCITITOR.md) +- [Policy Engine architecture](policy-engine.md) +- [Authority service](../ARCHITECTURE_AUTHORITY.md) +- [Observability standards (upcoming)](../observability/policy.md) – interim reference for telemetry naming. + +--- + +## 6 · Compliance checklist + +- [ ] AOC guard enabled for all Concelier and Excititor write paths in production. +- [ ] Mongo schema validators deployed for `advisory_raw` and `vex_raw`; change streams scoped per tenant. +- [ ] Authority scopes (`advisory:*`, `vex:*`, `effective:*`) configured in Gateway and validated via integration tests. +- [ ] `stella aoc verify` wired into CI/CD pipelines with seeded violation fixtures. +- [ ] Console AOC dashboard and CLI documentation reference the new ingestion contract. +- [ ] Offline Kit bundles include guard configs, verifier tooling, and documentation updates. +- [ ] Observability dashboards include violation, latency, and supersedes depth metrics with alert thresholds. + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/architecture/policy-engine.md b/docs/architecture/policy-engine.md new file mode 100644 index 00000000..8a854670 --- /dev/null +++ b/docs/architecture/policy-engine.md @@ -0,0 +1,243 @@ +# Policy Engine Architecture (v2) + +> **Ownership:** Policy Guild • Platform Guild +> **Services:** `StellaOps.Policy.Engine` (Minimal API + worker host) +> **Data Stores:** MongoDB (`policies`, `policy_runs`, `effective_finding_*`), Object storage (explain bundles), optional NATS/Mongo queue +> **Related docs:** [Policy overview](../policy/overview.md), [DSL](../policy/dsl.md), [Lifecycle](../policy/lifecycle.md), [Runs](../policy/runs.md), [REST API](../api/policy.md), [Policy CLI](../cli/policy.md), [Architecture overview](../architecture/overview.md), [AOC reference](../ingestion/aggregation-only-contract.md) + +This dossier describes the internal structure of the Policy Engine service delivered in Epic 2. It focuses on module boundaries, deterministic evaluation, orchestration, and integration contracts with Concelier, Excititor, SBOM Service, Authority, Scheduler, and Observability stacks. + +The service operates strictly downstream of the **Aggregation-Only Contract (AOC)**. It consumes immutable `advisory_raw` and `vex_raw` documents emitted by Concelier and Excititor, derives findings inside Policy-owned collections, and never mutates ingestion stores. Refer to the architecture overview and AOC reference for system-wide guardrails and provenance obligations. + +--- + +## 1 · Responsibilities & Constraints + +- Compile and evaluate `stella-dsl@1` policy packs into deterministic verdicts. +- Join SBOM inventory, Concelier advisories, and Excititor VEX evidence via canonical linksets and equivalence tables. +- Materialise effective findings (`effective_finding_{policyId}`) with append-only history and produce explain traces. +- Operate incrementally: react to change streams (advisory/vex/SBOM deltas) with ≤ 5 min SLA. +- Provide simulations with diff summaries for UI/CLI workflows without modifying state. +- Enforce strict determinism guard (no wall-clock, RNG, network beyond allow-listed services) and RBAC + tenancy via Authority scopes. +- Support sealed/air-gapped deployments with offline bundles and sealed-mode hints. + +Non-goals: policy authoring UI (handled by Console), ingestion or advisory normalisation (Concelier), VEX consensus (Excititor), runtime enforcement (Zastava). + +--- + +## 2 · High-Level Architecture + +```mermaid +graph TD + subgraph Clients + CLI[stella CLI] + UI[Console Policy Editor] + CI[CI Pipelines] + end + subgraph PolicyEngine["StellaOps.Policy.Engine"] + API[Minimal API Host] + Orchestrator[Run Orchestrator] + WorkerPool[Evaluation Workers] + Compiler[DSL Compiler Cache] + Materializer[Effective Findings Writer] + end + subgraph RawStores["Raw Stores (AOC)"] + AdvisoryRaw[(MongoDB
advisory_raw)] + VexRaw[(MongoDB
vex_raw)] + end + subgraph Derived["Derived Stores"] + Mongo[(MongoDB
policies / policy_runs / effective_finding_*)] + Blob[(Object Store / Evidence Locker)] + Queue[(Mongo Queue / NATS)] + end + Concelier[(Concelier APIs)] + Excititor[(Excititor APIs)] + SBOM[(SBOM Service)] + Authority[(Authority / DPoP Gateway)] + + CLI --> API + UI --> API + CI --> API + API --> Compiler + API --> Orchestrator + Orchestrator --> Queue + Queue --> WorkerPool + Concelier --> AdvisoryRaw + Excititor --> VexRaw + WorkerPool --> AdvisoryRaw + WorkerPool --> VexRaw + WorkerPool --> SBOM + WorkerPool --> Materializer + Materializer --> Mongo + WorkerPool --> Blob + API --> Mongo + API --> Blob + API --> Authority + Orchestrator --> Mongo + Authority --> API +``` + +Key notes: + +- API host exposes lifecycle, run, simulate, findings endpoints with DPoP-bound OAuth enforcement. +- Orchestrator manages run scheduling/fairness; writes run tickets to queue, leases jobs to worker pool. +- Workers evaluate policies using cached IR; join external services via tenant-scoped clients; pull immutable advisories/VEX from the raw stores; write derived overlays to Mongo and optional explain bundles to blob storage. +- Observability (metrics/traces/logs) integrated via OpenTelemetry (not shown). + +--- + +### 2.1 · AOC inputs & immutability + +- **Raw-only reads.** Evaluation workers access `advisory_raw` / `vex_raw` via tenant-scoped Mongo clients or the Concelier/Excititor raw APIs. No Policy Engine component is permitted to mutate these collections. +- **Guarded ingestion.** `AOCWriteGuard` rejects forbidden fields before data reaches the raw stores. Policy tests replay known `ERR_AOC_00x` violations to confirm ingestion compliance. +- **Change streams as contract.** Run orchestration stores resumable cursors for raw change streams. Replays of these cursors (e.g., after failover) must yield identical materialisation outcomes. +- **Derived stores only.** All severity, consensus, and suppression state lives in `effective_finding_*` collections and explain bundles owned by Policy Engine. Provenance fields link back to raw document IDs so auditors can trace every verdict. +- **Authority scopes.** Only the Policy Engine service identity holds `effective:write`. Ingestion identities retain `advisory:*`/`vex:*` scopes, ensuring separation of duties enforced by Authority and the API Gateway. + +--- + +## 3 · Module Breakdown + +| Module | Responsibility | Notes | +|--------|----------------|-------| +| **Configuration** (`Configuration/`) | Bind settings (Mongo URIs, queue options, service URLs, sealed mode), validate on start. | Strict schema; fails fast on missing secrets. | +| **Authority Client** (`Authority/`) | Acquire tokens, enforce scopes, perform DPoP key rotation. | Only service identity uses `effective:write`. | +| **DSL Compiler** (`Dsl/`) | Parse, canonicalise, IR generation, checksum caching. | Uses Roslyn-like pipeline; caches by `policyId+version+hash`. | +| **Selection Layer** (`Selection/`) | Batch SBOM ↔ advisory ↔ VEX joiners; apply equivalence tables; support incremental cursors. | Deterministic ordering (SBOM → advisory → VEX). | +| **Evaluator** (`Evaluation/`) | Execute IR with first-match semantics, compute severity/trust/reachability weights, record rule hits. | Stateless; all inputs provided by selection layer. | +| **Materialiser** (`Materialization/`) | Upsert effective findings, append history, manage explain bundle exports. | Mongo transactions per SBOM chunk. | +| **Orchestrator** (`Runs/`) | Change-stream ingestion, fairness, retry/backoff, queue writer. | Works with Scheduler Models DTOs. | +| **API** (`Api/`) | Minimal API endpoints, DTO validation, problem responses, idempotency. | Generated clients for CLI/UI. | +| **Observability** (`Telemetry/`) | Metrics (`policy_run_seconds`, `rules_fired_total`), traces, structured logs. | Sampled rule-hit logs with redaction. | +| **Offline Adapter** (`Offline/`) | Bundle export/import (policies, simulations, runs), sealed-mode enforcement. | Uses DSSE signing via Signer service. | + +--- + +## 4 · Data Model & Persistence + +### 4.1 Collections + +- `policies` – policy versions, metadata, lifecycle states, simulation artefact references. +- `policy_runs` – run records, inputs (cursors, env), stats, determinism hash, run status. +- `policy_run_events` – append-only log (queued, leased, completed, failed, canceled, replay). +- `effective_finding_{policyId}` – current verdict snapshot per finding. +- `effective_finding_{policyId}_history` – append-only history (previous verdicts, timestamps, runId). +- `policy_reviews` – review comments/decisions. + +### 4.2 Schema Highlights + +- Run records include `changeDigests` (hash of advisory/VEX inputs) for replay verification. +- Effective findings store provenance references (`advisory_raw_ids`, `vex_raw_ids`, `sbom_component_id`). +- All collections include `tenant`, `policyId`, `version`, `createdAt`, `updatedAt`, `traceId` for audit. + +### 4.3 Indexing + +- Compound indexes: `{tenant, policyId, status}` on `policies`; `{tenant, policyId, status, startedAt}` on `policy_runs`; `{policyId, sbomId, findingKey}` on findings. +- TTL indexes on transient explain bundle references (configurable). + +--- + +## 5 · Evaluation Pipeline + +```mermaid +sequenceDiagram + autonumber + participant Worker as EvaluationWorker + participant Compiler as CompilerCache + participant Selector as SelectionLayer + participant Eval as Evaluator + participant Mat as Materialiser + participant Expl as ExplainStore + + Worker->>Compiler: Load IR (policyId, version, digest) + Compiler-->>Worker: CompiledPolicy (cached or compiled) + Worker->>Selector: Fetch tuple batches (sbom, advisory, vex) + Selector-->>Worker: Deterministic batches (1024 tuples) + loop For each batch + Worker->>Eval: Execute rules (batch, env) + Eval-->>Worker: Verdicts + rule hits + Worker->>Mat: Upsert effective findings + Mat-->>Worker: Success + Worker->>Expl: Persist sampled explain traces (optional) + end + Worker->>Mat: Append history + run stats + Worker-->>Worker: Compute determinism hash + Worker->>+Mat: Finalize transaction + Mat-->>Worker: Ack +``` + +Determinism guard instrumentation wraps the evaluator, rejecting access to forbidden APIs and ensuring batch ordering remains stable. + +--- + +## 6 · Run Orchestration & Incremental Flow + +- **Change streams:** Concelier and Excititor publish document changes to the scheduler queue (`policy.trigger.delta`). Payload includes `tenant`, `source`, `linkset digests`, `cursor`. +- **Orchestrator:** Maintains per-tenant backlog; merges deltas until time/size thresholds met, then enqueues `PolicyRunRequest`. +- **Queue:** Mongo queue with lease; each job assigned `leaseDuration`, `maxAttempts`. +- **Workers:** Lease jobs, execute evaluation pipeline, report status (success/failure/canceled). Failures with recoverable errors requeue with backoff; determinism or schema violations mark job `failed` and raise incident event. +- **Fairness:** Round-robin per `{tenant, policyId}`; emergency jobs (`priority=emergency`) jump queue but limited via circuit breaker. +- **Replay:** On demand, orchestrator rehydrates run via stored cursors and exports sealed bundle for audit/CI determinism checks. + +--- + +## 7 · Security & Tenancy + +- **Auth:** All API calls pass through Authority gateway; DPoP tokens enforced for service-to-service (Policy Engine service principal). CLI/UI tokens include scope claims. +- **Scopes:** Mutations require `policy:*` scopes corresponding to action; `effective:write` restricted to service identity. +- **Tenancy:** All queries filter by `tenant`. Service identity uses `tenant-global` for shared policies; cross-tenant reads prohibited unless `policy:tenant-admin` scope present. +- **Secrets:** Configuration loaded via environment variables or sealed secrets; runtime avoids writing secrets to logs. +- **Determinism guard:** Static analyzer prevents referencing forbidden namespaces; runtime guard intercepts `DateTime.Now`, `Random`, `Guid`, HTTP clients beyond allow-list. +- **Sealed mode:** Global flag disables outbound network except allow-listed internal hosts; watchers fail fast if unexpected egress attempted. + +--- + +## 8 · Observability + +- Metrics: + - `policy_run_seconds{mode,tenant,policy}` (histogram) + - `policy_run_queue_depth{tenant}` + - `policy_rules_fired_total{policy,rule}` + - `policy_vex_overrides_total{policy,vendor}` +- Logs: Structured JSON with `traceId`, `policyId`, `version`, `runId`, `tenant`, `phase`. Guard ensures no sensitive data leakage. +- Traces: Spans `policy.select`, `policy.evaluate`, `policy.materialize`, `policy.simulate`. Trace IDs surfaced to CLI/UI. +- Incident mode toggles 100 % sampling and extended retention windows. + +--- + +## 9 · Offline / Bundle Integration + +- **Imports:** Offline Kit delivers policy packs, advisory/VEX snapshots, SBOM updates. Policy Engine ingests bundles via `offline import`. +- **Exports:** `stella policy bundle export` packages policy, IR digest, simulations, run metadata; UI provides export triggers. +- **Sealed hints:** Explain traces annotate when cached values used (EPSS, KEV). Run records mark `env.sealed=true`. +- **Sync cadence:** Operators perform monthly bundle sync; Policy Engine warns when snapshots > configured staleness (default 14 days). + +--- + +## 10 · Testing & Quality + +- **Unit tests:** DSL parsing, evaluator semantics, guard enforcement. +- **Integration tests:** Joiners with sample SBOM/advisory/VEX data; materialisation with deterministic ordering; API contract tests generated from OpenAPI. +- **Property tests:** Ensure rule evaluation deterministic across permutations. +- **Golden tests:** Replay recorded runs, compare determinism hash. +- **Performance tests:** Evaluate 100k component / 1M advisory dataset under warmed caches (<30 s full run). +- **Chaos hooks:** Optional toggles to simulate upstream latency/failures; used in staging. + +--- + +## 11 · Compliance Checklist + +- [ ] **Determinism guard enforced:** Static analyzer + runtime guard block wall-clock, RNG, unauthorized network calls. +- [ ] **Incremental correctness:** Change-stream cursors stored and replayed during tests; unit/integration coverage for dedupe. +- [ ] **RBAC validated:** Endpoint scope requirements match Authority configuration; integration tests cover deny/allow. +- [ ] **AOC separation enforced:** No code path writes to `advisory_raw` / `vex_raw`; integration tests capture `ERR_AOC_00x` handling; read-only clients verified. +- [ ] **Effective findings ownership:** Only Policy Engine identity holds `effective:write`; unauthorized callers receive `ERR_AOC_006`. +- [ ] **Observability wired:** Metrics/traces/logs exported with correlation IDs; dashboards include `aoc_violation_total` and ingest latency panels. +- [ ] **Offline parity:** Sealed-mode tests executed; bundle import/export flows documented and validated. +- [ ] **Schema docs synced:** DTOs match Scheduler Models (`SCHED-MODELS-20-001`); JSON schemas committed. +- [ ] **Security reviews complete:** Threat model (including queue poisoning, determinism bypass, data exfiltration) documented; mitigations in place. +- [ ] **Disaster recovery rehearsed:** Run replay+rollback procedures tested and recorded. + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/ci/20_CI_RECIPES.md b/docs/ci/20_CI_RECIPES.md index dd17e6dc..2023b0a6 100755 --- a/docs/ci/20_CI_RECIPES.md +++ b/docs/ci/20_CI_RECIPES.md @@ -8,7 +8,9 @@ | `DOCKER_HOST` | How containers reach your Docker daemon (because we no longer mount `/var/run/docker.sock`) | `tcp://docker:2375` | | `WORKSPACE` | Directory where the pipeline stores artefacts (SBOM file) | `$(pwd)` | | `IMAGE` | The image you are building & scanning | `acme/backend:sha-${COMMIT_SHA}` | -| `SBOM_FILE` | Immutable SBOM name – `‑YYYYMMDDThhmmssZ.sbom.json` | `acme_backend_sha‑abc123‑20250804T153050Z.sbom.json` | +| `SBOM_FILE` | Immutable SBOM name – `‑YYYYMMDDThhmmssZ.sbom.json` | `acme_backend_sha‑abc123‑20250804T153050Z.sbom.json` | + +> **Authority graph scopes note (2025‑10‑27):** CI stages that spin up the Authority compose profile now rely on the checked-in `etc/authority.yaml`. Before running integration smoke jobs, inject real secrets for every `etc/secrets/*.secret` file (Cartographer, Graph API, Policy Engine, Concelier, Excititor). The repository defaults contain `*-change-me` placeholders and Authority will reject tokens if those secrets are not overridden. ```bash export STELLA_URL="stella-ops.ci.acme.example" @@ -291,6 +293,40 @@ Host the resulting bundle via any static file server for review (for example `py - [ ] Markdown link check (`npx markdown-link-check`) reports no broken references. - [ ] Preview bundle archived (or attached) for stakeholders. +### 4.5 Policy DSL lint stage + +Policy Engine v2 pipelines now fail fast if policy documents are malformed. After checkout and dotnet restore, run: + +```bash +dotnet run \ + --project tools/PolicyDslValidator/PolicyDslValidator.csproj \ + -- \ + --strict docs/examples/policies/*.yaml +``` + +- `--strict` treats warnings as errors so missing metadata doesn’t slip through. +- The validator accepts globs, so you can point it at tenant policy directories later (`policies/**/*.yaml`). +- Exit codes follow UNIX conventions: `0` success, `1` parse/errors, `2` warnings when `--strict` is set, `64` usage mistakes. + +Capture the validator output as part of your build logs; Support uses it when triaging policy rollout issues. + +### 4.6 Policy simulation smoke + +Catch unexpected policy regressions by exercising a small set of golden SBOM findings via the simulation smoke tool: + +```bash +dotnet run \ + --project tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj \ + -- \ + --scenario-root samples/policy/simulations \ + --output artifacts/policy-simulations +``` + +- The tool loads each `scenario.json` under `samples/policy/simulations`, evaluates the referenced policy, and fails the build if projected verdicts change. +- In CI the command runs twice (to `run1/` and `run2/`) and `diff -u` compares the summaries—any mismatch signals a determinism regression. +- Artifacts land in `artifacts/policy-simulations/policy-simulation-summary.json`; upload them for later inspection (see CI workflow). +- Expand scenarios by copying real-world findings into the samples directory—ensure expected statuses are recorded so regressions trip the pipeline. + --- ## 5 · Troubleshooting cheat‑sheet diff --git a/docs/cli/cli-reference.md b/docs/cli/cli-reference.md new file mode 100644 index 00000000..17d8c4de --- /dev/null +++ b/docs/cli/cli-reference.md @@ -0,0 +1,284 @@ +# CLI AOC Commands Reference + +> **Audience:** DevEx engineers, operators, and CI authors integrating the `stella` CLI with Aggregation-Only Contract (AOC) workflows. +> **Scope:** Command synopsis, options, exit codes, and offline considerations for `stella sources ingest --dry-run` and `stella aoc verify` as introduced in Sprint 19. + +Both commands are designed to enforce the AOC guardrails documented in the [aggregation-only reference](../ingestion/aggregation-only-contract.md) and the [architecture overview](../architecture/overview.md). They consume Authority-issued tokens with tenant scopes and never mutate ingestion stores. + +--- + +## 1 · Prerequisites + +- CLI version: `stella` ≥ 0.19.0 (AOC feature gate enabled). +- Required scopes (DPoP-bound): + - `advisory:verify` for Concelier sources. + - `vex:verify` for Excititor sources (optional but required for VEX checks). + - `tenant:select` if your deployment uses tenant switching. +- Connectivity: direct access to Concelier/Excititor APIs or Offline Kit snapshot (see § 4). +- Environment: set `STELLA_AUTHORITY_URL`, `STELLA_TENANT`, and export a valid OpTok via `stella auth login` or existing token cache. + +--- + +## 2 · `stella sources ingest --dry-run` + +### 2.1 Synopsis + +```bash +stella sources ingest --dry-run \ + --source \ + --input \ + [--tenant ] \ + [--format json|table] \ + [--no-color] \ + [--output ] +``` + +### 2.2 Description + +Previews an ingestion write without touching MongoDB. The command loads an upstream advisory or VEX document, computes the would-write payload, runs it through the `AOCWriteGuard`, and reports any forbidden fields, provenance gaps, or idempotency issues. Use it during connector development, CI validation, or while triaging incidents. + +### 2.3 Options + +| Option | Description | +|--------|-------------| +| `--source ` | Logical source name (`redhat`, `ubuntu`, `osv`, etc.). Mirrors connector configuration. | +| `--input ` | Path to local CSAF/OSV/VEX file or HTTPS URI. CLI normalises transport (gzip/base64) before guard evaluation. | +| `--tenant ` | Overrides default tenant for multi-tenant deployments. Mandatory when `STELLA_TENANT` is not set. | +| `--format json|table` | Output format. `table` (default) prints summary with highlighted violations; `json` emits machine-readable report (see below). | +| `--no-color` | Disables ANSI colour output for CI logs. | +| `--output ` | Writes the JSON report to file while still printing human-readable summary to stdout. | + +### 2.4 Output schema (JSON) + +```json +{ + "source": "redhat", + "tenant": "default", + "guardVersion": "1.0.0", + "status": "ok", + "document": { + "contentHash": "sha256:…", + "supersedes": null, + "provenance": { + "signature": { "format": "pgp", "present": true } + } + }, + "violations": [] +} +``` + +When violations exist, `status` becomes `error` and `violations` contains entries with `code` (`ERR_AOC_00x`), a short `message`, and JSON Pointer `path` values indicating offending fields. + +### 2.5 Exit codes + +| Exit code | Meaning | +|-----------|---------| +| `0` | Guard passed; would-write payload is AOC compliant. | +| `11` | `ERR_AOC_001` – Forbidden field (`severity`, `cvss`, etc.) detected. | +| `12` | `ERR_AOC_002` – Merge attempt (multiple upstream sources fused). | +| `13` | `ERR_AOC_003` – Idempotency violation (duplicate without supersedes). | +| `14` | `ERR_AOC_004` – Missing provenance fields. | +| `15` | `ERR_AOC_005` – Signature/checksum mismatch. | +| `16` | `ERR_AOC_006` – Effective findings present (Policy-only data). | +| `17` | `ERR_AOC_007` – Unknown top-level fields / schema violation. | +| `70` | Transport error (network, auth, malformed input). | + +> Exit codes map directly to the `ERR_AOC_00x` table for scripting consistency. Multiple violations yield the highest-priority code (e.g., 11 takes precedence over 14). + +### 2.6 Examples + +Dry-run a local CSAF file: + +```bash +stella sources ingest --dry-run \ + --source redhat \ + --input ./fixtures/redhat/RHSA-2025-1234.json +``` + +Stream from HTTPS and emit JSON for CI: + +```bash +stella sources ingest --dry-run \ + --source osv \ + --input https://osv.dev/vulnerability/GHSA-aaaa-bbbb \ + --format json \ + --output artifacts/osv-dry-run.json + +cat artifacts/osv-dry-run.json | jq '.violations' +``` + +### 2.7 Offline notes + +When operating in sealed/offline mode: + +- Use `--input` paths pointing to Offline Kit snapshots (`offline-kit/advisories/*.json`). +- Provide `--tenant` explicitly if the offline bundle contains multiple tenants. +- The command does not attempt network access when given a file path. +- Store reports with `--output` to include in transfer packages for policy review. + +--- + +## 3 · `stella aoc verify` + +### 3.1 Synopsis + +```bash +stella aoc verify \ + [--since ] \ + [--limit ] \ + [--sources ] \ + [--codes ] \ + [--format table|json] \ + [--export ] \ + [--tenant ] \ + [--no-color] +``` + +### 3.2 Description + +Replays the AOC guard against stored raw documents. By default it checks all advisories and VEX statements ingested in the last 24 hours for the active tenant, reporting totals, top violation codes, and sample documents. Use it in CI pipelines, scheduled verifications, or during incident response. + +### 3.3 Options + +| Option | Description | +|--------|-------------| +| `--since ` | Verification window. Accepts ISO 8601 timestamp (`2025-10-25T12:00:00Z`) or duration (`48h`, `7d`). Defaults to `24h`. | +| `--limit ` | Maximum number of violations to display (per code). `0` means show all. Defaults to `20`. | +| `--sources ` | Comma-separated list of sources (`redhat,ubuntu,osv`). Filters both advisories and VEX entries. | +| `--codes ` | Restricts output to specific `ERR_AOC_00x` codes. Useful for regression tracking. | +| `--format table|json` | `table` (default) prints summary plus top violations; `json` outputs machine-readable report identical to the `/aoc/verify` API. | +| `--export ` | Writes the JSON report to disk (useful for audits/offline uploads). | +| `--tenant ` | Overrides tenant context. Required for cross-tenant verifications when run by platform operators. | +| `--no-color` | Disables ANSI colours. | + +### 3.4 Report structure (JSON) + +```json +{ + "tenant": "default", + "window": { + "from": "2025-10-25T12:00:00Z", + "to": "2025-10-26T12:00:00Z" + }, + "checked": { + "advisories": 482, + "vex": 75 + }, + "violations": [ + { + "code": "ERR_AOC_001", + "count": 2, + "examples": [ + { + "source": "redhat", + "documentId": "advisory_raw:redhat:RHSA-2025:1", + "contentHash": "sha256:…", + "path": "/content/raw/cvss" + } + ] + } + ], + "metrics": { + "ingestion_write_total": 557, + "aoc_violation_total": 2 + } +} +``` + +### 3.5 Exit codes + +| Exit code | Meaning | +|-----------|---------| +| `0` | Verification succeeded with zero violations. | +| `11…17` | Same mapping as § 2.5 when violations are detected. Highest-priority code returned. | +| `18` | Verification ran but results truncated (limit reached) – treat as warning; rerun with higher `--limit`. | +| `70` | Transport/authentication error. | +| `71` | CLI misconfiguration (missing tenant, invalid `--since`, etc.). | + +### 3.6 Examples + +Daily verification across all sources: + +```bash +stella aoc verify --since 24h --format table +``` + +CI pipeline focusing on errant sources and exporting evidence: + +```bash +stella aoc verify \ + --sources redhat,ubuntu \ + --codes ERR_AOC_001,ERR_AOC_004 \ + --format json \ + --limit 100 \ + --export artifacts/aoc-verify.json + +jq '.violations[] | {code, count}' artifacts/aoc-verify.json +``` + +Air-gapped verification using Offline Kit snapshot (example script): + +```bash +stella aoc verify \ + --since 7d \ + --format json \ + --export /mnt/offline/aoc-verify-$(date +%F).json + +sha256sum /mnt/offline/aoc-verify-*.json > /mnt/offline/checksums.txt +``` + +### 3.7 Automation tips + +- Schedule with `cron` or platform scheduler and fail the job when exit code ≥ 11. +- Pair with `stella sources ingest --dry-run` for pre-flight validation before re-enabling a paused source. +- Push JSON exports to observability pipelines for historical tracking of violation counts. + +### 3.8 Offline notes + +- Works against Offline Kit Mongo snapshots when CLI is pointed at the local API gateway included in the bundle. +- When fully disconnected, run against exported `aoc verify` reports generated on production and replay them using `--format json --export` (automation recipe above). +- Include verification output in compliance packages alongside Offline Kit manifests. + +--- + +## 4 · Global exit-code reference + +| Code | Summary | +|------|---------| +| `0` | Success / no violations. | +| `11` | `ERR_AOC_001` – Forbidden field present. | +| `12` | `ERR_AOC_002` – Merge attempt detected. | +| `13` | `ERR_AOC_003` – Idempotency violation. | +| `14` | `ERR_AOC_004` – Missing provenance/signature metadata. | +| `15` | `ERR_AOC_005` – Signature/checksum mismatch. | +| `16` | `ERR_AOC_006` – Effective findings in ingestion payload. | +| `17` | `ERR_AOC_007` – Schema violation / unknown fields. | +| `18` | Partial verification (limit reached). | +| `70` | Transport or HTTP failure. | +| `71` | CLI usage error (invalid arguments, missing tenant). | + +Use these codes in CI to map outcomes to build statuses or alert severities. + +--- + +## 5 · Related references + +- [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) +- [Architecture overview](../architecture/overview.md) +- [Console AOC dashboard](../ui/console.md) +- [Authority scopes](../ARCHITECTURE_AUTHORITY.md) + +--- + +## 6 · Compliance checklist + +- [ ] Usage documented for both table and JSON formats. +- [ ] Exit-code mapping matches `ERR_AOC_00x` definitions and automation guidance. +- [ ] Offline/air-gap workflow captured for both commands. +- [ ] References to AOC architecture and console docs included. +- [ ] Examples validated against current CLI syntax (update post-implementation). +- [ ] Docs guild screenshot/narrative placeholder logged for release notes (pending CLI team capture). + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/cli/policy.md b/docs/cli/policy.md new file mode 100644 index 00000000..fc4976f0 --- /dev/null +++ b/docs/cli/policy.md @@ -0,0 +1,284 @@ +# Stella CLI — Policy Commands + +> **Audience:** Policy authors, reviewers, operators, and CI engineers using the `stella` CLI to interact with Policy Engine. +> **Supported from:** `stella` CLI ≥ 0.20.0 (Policy Engine v2 sprint line). +> **Prerequisites:** Authority-issued bearer token with the scopes noted per command (export `STELLA_TOKEN` or pass `--token`). + +--- + +## 1 · Global Options & Output Modes + +All `stella policy *` commands honour the common CLI options: + +| Flag | Default | Description | +|------|---------|-------------| +| `--server ` | `https://stella.local` | Policy Engine gateway root. | +| `--tenant ` | token default | Override tenant for multi-tenant installs. | +| `--format ` | `table` for TTY, `json` otherwise | Output format for listings/diffs. | +| `--output ` | stdout | Write full JSON payload to file. | +| `--sealed` | false | Force sealed-mode behaviour (no outbound fetch). | +| `--trace` | false | Emit verbose timing/log correlation info. | + +> **Tip:** Set `STELLA_PROFILE=policy` in CI to load saved defaults from `~/.stella/profiles/policy.toml`. + +--- + +## 2 · Authoring & Drafting Commands + +### 2.1 `stella policy new` + +Create a draft policy from a template or scratch. + +``` +stella policy new --policy-id P-7 --name "Default Org Policy" \ + --template baseline --output-path policies/P-7.stella +``` + +Options: + +| Flag | Description | +|------|-------------| +| `--policy-id` *(required)* | Stable identifier (e.g., `P-7`). | +| `--name` | Friendly display name. | +| `--template` | `baseline`, `serverless`, `blank`. | +| `--from` | Start from existing version (`policyId@version`). | +| `--open` | Launches `$EDITOR` after creation. | + +Writes DSL to local file and registers draft version (`status=draft`). Requires `policy:write`. + +### 2.2 `stella policy edit` + +Open an existing draft in the local editor. + +``` +stella policy edit P-7 --version 4 +``` + +- Auto-checks out latest draft if `--version` omitted. +- Saves to temp file, uploads on editor exit (unless `--no-upload`). +- Use `--watch` to keep command alive and re-upload on every save. + +### 2.3 `stella policy lint` + +Static validation without submitting. + +``` +stella policy lint policies/P-7.stella --format json +``` + +Outputs diagnostics (line/column, code, message). Exit codes: + +| Code | Meaning | +|------|---------| +| `0` | No lint errors. | +| `10` | Syntax/compile errors (`ERR_POL_001`). | +| `11` | Unsupported syntax version. | + +### 2.4 `stella policy compile` + +Emits IR digest and rule summary. + +``` +stella policy compile P-7 --version 4 +``` + +Returns JSON with `digest`, `rules.count`, action counts. Exit `0` success, `10` on compile errors. + +--- + +## 3 · Lifecycle Workflow + +### 3.1 Submit + +``` +stella policy submit P-7 --version 4 \ + --reviewer user:kay --reviewer group:sec-reviewers \ + --note "Simulated against golden SBOM set" \ + --attach sims/P-7-v4-vs-v3.json +``` + +Requires `policy:submit`. CLI validates that lint/compile run within 24 h and bundle attachments exist. + +### 3.2 Review + +``` +stella policy review P-7 --version 4 --approve \ + --note "Looks good; ensure incident playbook updated." +``` + +- `--approve`, `--request-changes`, or `--comment`. +- Provide `--blocking` to mark comment as blocking. +- Requires `policy:review`. + +### 3.3 Approve + +``` +stella policy approve P-7 --version 4 \ + --note "Determinism CI green; simulation diff attached." \ + --attach sims/P-7-v4-vs-v3.json +``` + +Prompts for confirmation; refuses if approver == submitter. Requires `policy:approve`. + +### 3.4 Activate + +``` +stella policy activate P-7 --version 4 --run-now --priority high +``` + +- Optional `--scheduled-at 2025-10-27T02:00:00Z`. +- Requires `policy:activate` and `policy:run`. + +### 3.5 Archive / Rollback + +``` +stella policy archive P-7 --version 3 --reason "Superseded by v4" +stella policy activate P-7 --version 3 --rollback --incident INC-2025-104 +``` + +--- + +## 4 · Simulation & Runs + +### 4.1 Simulate + +``` +stella policy simulate P-7 \ + --base 3 --candidate 4 \ + --sbom sbom:S-42 --sbom sbom:S-318 \ + --env exposure=internet --env sealed=false \ + --format json --output sims/P-7-v4-vs-v3.json +``` + +Output fields (JSON): + +```json +{ + "diff": { + "added": 12, + "removed": 8, + "unchanged": 657, + "bySeverity": { + "Critical": {"up": 1, "down": 0}, + "High": {"up": 3, "down": 4} + } + }, + "explainUri": "blob://policy/P-7/simulations/2025-10-26.json" +} +``` + +> Schema reminder: CLI commands surface objects defined in `src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md`; use the samples in `samples/api/scheduler/` for contract validation when extending output parsing. + +Exit codes: + +| Code | Meaning | +|------|---------| +| `0` | Simulation succeeded; diffs informational. | +| `20` | Blocking delta (`--fail-on-diff` triggered). | +| `21` | Simulation input missing (`ERR_POL_003`). | +| `22` | Determinism guard (`ERR_POL_004`). | +| `23` | API/permission error (`ERR_POL_002`, `ERR_POL_005`). | + +### 4.2 Run + +``` +stella policy run P-7 --mode full \ + --sbom sbom:S-42 --env exposure=internal-only \ + --wait --watch +``` + +Options: + +| Flag | Description | +|------|-------------| +| `--mode` | `full` or `incremental` (default incremental). | +| `--sbom` | Explicit SBOM IDs (optional). | +| `--priority` | `normal`, `high`, `emergency`. | +| `--wait` | Poll run status until completion. | +| `--watch` | Stream progress events (requires TTY). | + +`stella policy run status ` retrieves run metadata. +`stella policy run list --status failed --limit 20` returns recent runs. + +### 4.3 Replay & Cancel + +``` +stella policy run replay run:P-7:2025-10-26:auto --output bundles/replay.tgz +stella policy run cancel run:P-7:2025-10-26:auto +``` + +Replay downloads sealed bundle for deterministic verification. + +--- + +## 5 · Findings & Explainability + +### 5.1 List Findings + +``` +stella findings ls --policy P-7 \ + --sbom sbom:S-42 \ + --status affected --severity High,Critical \ + --format table +``` + +Common flags: + +| Flag | Description | +|------|-------------| +| `--page`, `--page-size` | Pagination (default page size 50). | +| `--cursor` | Use cursor token from previous call. | +| `--since` | ISO timestamp filter. | + +### 5.2 Fetch Explain + +``` +stella findings explain --policy P-7 --finding P-7:S-42:pkg:npm/lodash@4.17.21:CVE-2021-23337 \ + --format json --output explains/lodash.json +``` + +Outputs ordered rule hits, inputs, and sealed-mode hints. + +--- + +## 6 · Exit Codes Summary + +| Exit code | Description | Typical ERR codes | +|-----------|-------------|-------------------| +| `0` | Success (command completed, warnings only). | — | +| `10` | DSL syntax/compile failure. | `ERR_POL_001` | +| `11` | Unsupported DSL version / schema mismatch. | `ERR_POL_001` | +| `12` | Approval/rbac failure. | `ERR_POL_002`, `ERR_POL_005` | +| `20` | Simulation diff exceeded thresholds (`--fail-on-diff`). | — | +| `21` | Required inputs missing (SBOM/advisory/VEX). | `ERR_POL_003` | +| `22` | Determinism guard triggered. | `ERR_POL_004` | +| `23` | Run canceled or timed out. | `ERR_POL_006` | +| `30` | Network/transport error (non-HTTP success). | — | +| `64` | CLI usage error (invalid flag/argument). | — | + +All non-zero exits emit structured error envelope on stderr when `--format json` or `STELLA_JSON_ERRORS=1`. + +--- + +## 7 · Offline & Air-Gap Usage + +- Use `--sealed` to ensure commands avoid outbound calls; required for sealed enclaves. +- `stella policy bundle export --policy P-7 --version 4 --output bundles/policy-P-7-v4.bundle` pairs with Offline Kit import. +- Replay bundles (`run replay`) are DSSE-signed; verify with `stella offline verify`. +- Store credentials in `~/.stella/offline.toml` for non-interactive air-gapped pipelines. + +--- + +## 8 · Compliance Checklist + +- [ ] **Help text synced:** `stella policy --help` matches documented flags/examples (update during release pipeline). +- [ ] **Exit codes mapped:** Table above reflects CLI implementation and CI asserts mapping for `ERR_POL_*`. +- [ ] **JSON schemas verified:** Example payloads validated against OpenAPI/SDK contracts before publishing. +- [ ] **Scope guidance present:** Each command lists required Authority scopes. +- [ ] **Offline guidance included:** Sealed-mode steps and bundle workflows documented. +- [ ] **Cross-links tested:** Links to DSL, lifecycle, runs, and API docs resolve locally (`yarn docs:lint`). +- [ ] **Examples no-op safe:** Command examples either read-only or use placeholders (no destructive defaults). + +--- + +*Last updated: 2025-10-26 (Sprint 20).* diff --git a/docs/deploy/console.md b/docs/deploy/console.md new file mode 100644 index 00000000..c93a292e --- /dev/null +++ b/docs/deploy/console.md @@ -0,0 +1,229 @@ +# Deploying the StellaOps Console + +> **Audience:** Deployment Guild, Console Guild, operators rolling out the web console. +> **Scope:** Helm and Docker Compose deployment steps, ingress/TLS configuration, required environment variables, health checks, offline/air-gap operation, and compliance checklist (Sprint 23). + +The StellaOps Console ships as part of the `stellaops` stack Helm chart and Compose bundles maintained under `deploy/`. This guide describes the supported deployment paths, the configuration surface, and operational checks needed to run the console in connected or air-gapped environments. + +--- + +## 1. Prerequisites + +- Kubernetes cluster (v1.28+) with ingress controller (NGINX, Traefik, or equivalent) and Cert-Manager for automated TLS, or Docker host for Compose deployments. +- Container registry access to `registry.stella-ops.org` (or mirrored registry) for all images listed in `deploy/releases/*.yaml`. +- Authority service configured with console client (`aud=ui`, scopes `ui.read`, `ui.admin`). +- DNS entry pointing to the console hostname (for example, `console.acme.internal`). +- Cosign public key for manifest verification (`deploy/releases/manifest.json.sig`). +- Optional: Offline Kit bundle for air-gapped sites (`stella-ops-offline-kit-.tar.gz`). + +--- + +## 2. Helm deployment (recommended) + +### 2.1 Install chart repository + +```bash +helm repo add stellaops https://downloads.stella-ops.org/helm +helm repo update stellaops +``` + +If operating offline, copy the chart archive from the Offline Kit (`deploy/helm/stellaops-.tgz`) and run: + +```bash +helm install stellaops ./stellaops-.tgz --namespace stellaops --create-namespace +``` + +### 2.2 Base installation + +```bash +helm install stellaops stellaops/stellaops \ + --namespace stellaops \ + --create-namespace \ + --values deploy/helm/stellaops/values-prod.yaml +``` + +The chart deploys Authority, Console web/API gateway, Scanner API, Scheduler, and supporting services. The console frontend pod is labelled `app=stellaops-web-ui`. + +### 2.3 Helm values highlights + +Key sections in `deploy/helm/stellaops/values-prod.yaml`: + +| Path | Description | +|------|-------------| +| `console.ingress.host` | Hostname served by the console (`console.example.com`). | +| `console.ingress.tls.secretName` | Kubernetes secret containing TLS certificate (generated by Cert-Manager or uploaded manually). | +| `console.config.apiGateway.baseUrl` | Internal base URL the UI uses to reach the gateway (defaults to `https://stellaops-web`). | +| `console.env.AUTHORITY_ISSUER` | Authority issuer URL (for example, `https://authority.example.com`). | +| `console.env.AUTHORITY_CLIENT_ID` | Authority client ID for the console UI. | +| `console.env.AUTHORITY_SCOPES` | Space-separated scopes required by UI (`ui.read ui.admin`). | +| `console.resources` | CPU/memory requests and limits (default 250m CPU / 512Mi memory). | +| `console.podAnnotations` | Optional annotations for service mesh or monitoring. | + +Use `values-stage.yaml`, `values-dev.yaml`, or `values-airgap.yaml` as templates for other environments. + +### 2.4 TLS and ingress + +Example ingress override: + +```yaml +console: + ingress: + enabled: true + className: nginx + host: console.acme.internal + tls: + enabled: true + secretName: console-tls +``` + +Generate certificates using Cert-Manager or provide an existing secret. For air-gapped deployments, pre-create the secret with the mirrored CA chain. + +### 2.5 Health checks + +Console pods expose: + +| Path | Purpose | Notes | +|------|---------|-------| +| `/health/live` | Liveness probe | Confirms process responsive. | +| `/health/ready` | Readiness probe | Verifies configuration bootstrap and Authority reachability. | +| `/metrics` | Prometheus metrics | Enabled when `console.metrics.enabled=true`. | + +Helm chart sets default probes (`initialDelaySeconds: 10`, `periodSeconds: 15`). Adjust via `console.livenessProbe` and `console.readinessProbe`. + +--- + +## 3. Docker Compose deployment + +Located in `deploy/compose/docker-compose.console.yaml`. Quick start: + +```bash +cd deploy/compose +docker compose -f docker-compose.console.yaml --env-file console.env up -d +``` + +`console.env` should define: + +``` +CONSOLE_PUBLIC_BASE_URL=https://console.acme.internal +AUTHORITY_ISSUER=https://authority.acme.internal +AUTHORITY_CLIENT_ID=console-ui +AUTHORITY_CLIENT_SECRET= +AUTHORITY_SCOPES=ui.read ui.admin +CONSOLE_GATEWAY_BASE_URL=https://api.acme.internal +``` + +The compose bundle includes Traefik as reverse proxy with TLS termination. Update `traefik/dynamic/console.yml` for custom certificates or additional middlewares (CSP headers, rate limits). + +--- + +## 4. Environment variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `CONSOLE_PUBLIC_BASE_URL` | External URL used for redirects, deep links, and telemetry. | None (required). | +| `CONSOLE_GATEWAY_BASE_URL` | URL of the web gateway that proxies API calls (`/console/*`). | Chart service name. | +| `AUTHORITY_ISSUER` | Authority issuer (`https://authority.example.com`). | None (required). | +| `AUTHORITY_CLIENT_ID` | OIDC client configured in Authority. | None (required). | +| `AUTHORITY_SCOPES` | Space-separated scopes assigned to the console client. | `ui.read ui.admin`. | +| `AUTHORITY_DPOP_ENABLED` | Enables DPoP challenge/response (recommended true). | `true`. | +| `CONSOLE_FEATURE_FLAGS` | Comma-separated feature flags (`runs`, `downloads.offline`, etc.). | `runs,downloads,policies`. | +| `CONSOLE_LOG_LEVEL` | Minimum log level (`Information`, `Debug`, etc.). | `Information`. | +| `CONSOLE_METRICS_ENABLED` | Expose `/metrics` endpoint. | `true`. | +| `CONSOLE_SENTRY_DSN` | Optional error reporting DSN. | Blank. | + +When running behind additional proxies, set `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true` to honour `X-Forwarded-*` headers. + +--- + +## 5. Security headers and CSP + +The console serves a strict Content Security Policy (CSP) by default: + +``` +default-src 'self'; +connect-src 'self' https://*.stella-ops.local; +script-src 'self'; +style-src 'self' 'unsafe-inline'; +img-src 'self' data:; +font-src 'self'; +frame-ancestors 'none'; +``` + +Adjust via `console.config.cspOverrides` if additional domains are required. For integrations embedding the console, update OIDC redirect URIs and Authority scopes accordingly. + +TLS recommendations: + +- Use TLS 1.2+ with modern cipher suite policy. +- Enable HSTS (`Strict-Transport-Security: max-age=31536000; includeSubDomains`). +- Provide custom trust bundles via `console.config.trustBundleSecret` when using private CAs. + +--- + +## 6. Logging and metrics + +- Structured logs emitted to stdout with correlation IDs. Configure log shipping via Fluent Bit or similar. +- Metrics available at `/metrics` in Prometheus format. Key metrics include `ui_request_duration_seconds`, `ui_tenant_switch_total`, and `ui_download_manifest_refresh_seconds`. +- Enable OpenTelemetry exporter by setting `OTEL_EXPORTER_OTLP_ENDPOINT` and associated headers in environment variables. + +--- + +## 7. Offline and air-gap deployment + +- Mirror container images using the Downloads workspace or Offline Kit manifest. Example: + +```bash +oras copy registry.stella-ops.org/stellaops/web-ui@sha256: \ + registry.airgap.local/stellaops/web-ui:2025.10.0 +``` + +- Import Offline Kit using `stella ouk import` before starting the console so manifest parity checks succeed. +- Use `values-airgap.yaml` to disable external telemetry endpoints and configure internal certificate chains. +- Run `helm upgrade --install` using the mirrored chart (`stellaops-.tgz`) and set `console.offlineMode=true` to surface offline banners. + +--- + +## 8. Health checks and remediation + +| Check | Command | Expected result | +|-------|---------|-----------------| +| Pod status | `kubectl get pods -n stellaops` | `Running` state with restarts = 0. | +| Liveness | `kubectl exec deploy/stellaops-web-ui -- curl -fsS http://localhost:8080/health/live` | Returns `{"status":"Healthy"}`. | +| Readiness | `kubectl exec deploy/stellaops-web-ui -- curl -fsS http://localhost:8080/health/ready` | Returns `{"status":"Ready"}`. | +| Gateway reachability | `curl -I https://console.example.com/api/console/status` | `200 OK` with CSP headers. | +| Static assets | `curl -I https://console.example.com/static/assets/app.js` | `200 OK` with long cache headers. | + +Troubleshooting steps: + +- **Authority unreachable:** readiness fails with `AUTHORITY_UNREACHABLE`. Check DNS, trust bundles, and Authority service health. +- **Manifest mismatch:** console logs `DOWNLOAD_MANIFEST_SIGNATURE_INVALID`. Verify cosign key and re-sync manifest. +- **Ingress 404:** ensure ingress controller routes host to `stellaops-web-ui` service; check TLS secret name. +- **SSE blocked:** confirm proxy allows HTTP/1.1 and disables buffering on `/console/runs/*`. + +--- + +## 9. References + +- `deploy/helm/stellaops/values-*.yaml` - environment-specific overrides. +- `deploy/compose/docker-compose.console.yaml` - Compose bundle. +- `/docs/ui/downloads.md` - manifest and offline bundle guidance. +- `/docs/security/console-security.md` (pending) - CSP and Authority scopes. +- `/docs/24_OFFLINE_KIT.md` - Offline kit packaging and verification. +- `/docs/ops/deployment-runbook.md` (pending) - wider platform deployment steps. + +--- + +## 10. Compliance checklist + +- [ ] Helm and Compose instructions verified against `deploy/` assets. +- [ ] Ingress/TLS guidance aligns with Security Guild recommendations. +- [ ] Environment variables documented with defaults and required values. +- [ ] Health/liveness/readiness endpoints tested and listed. +- [ ] Offline workflow (mirrors, manifest parity) captured. +- [ ] Logging and metrics surface documented metrics. +- [ ] CSP and security header defaults stated alongside override guidance. +- [ ] Troubleshooting section linked to relevant runbooks. + +--- + +*Last updated: 2025-10-27 (Sprint 23).* + diff --git a/docs/deploy/containers.md b/docs/deploy/containers.md new file mode 100644 index 00000000..69ca137b --- /dev/null +++ b/docs/deploy/containers.md @@ -0,0 +1,135 @@ +# Container Deployment Guide — AOC Update + +> **Audience:** DevOps Guild, platform operators deploying StellaOps services. +> **Scope:** Deployment configuration changes required by the Aggregation-Only Contract (AOC), including schema validators, guard environment flags, and verifier identities. + +This guide supplements existing deployment manuals with AOC-specific configuration. It assumes familiarity with the base Compose/Helm manifests described in `ops/deployment/` and `docs/ARCHITECTURE_DEVOPS.md`. + +--- + +## 1 · Schema validator enablement + +### 1.1 MongoDB validators + +- Apply JSON schema validators to `advisory_raw` and `vex_raw` collections before enabling AOC guards. +- Use the migration script provided in `ops/devops/scripts/apply-aoc-validators.js`: + +```bash +kubectl exec -n concelier deploy/concelier-mongo -- \ + mongo concelier ops/devops/scripts/apply-aoc-validators.js + +kubectl exec -n excititor deploy/excititor-mongo -- \ + mongo excititor ops/devops/scripts/apply-aoc-validators.js +``` + +- Validators enforce required fields (`tenant`, `source`, `upstream`, `linkset`) and reject forbidden keys at DB level. +- Rollback plan: validators are applied with `validationLevel: moderate`—downgrade via the same script with `--remove` if required. + +### 1.2 Migration order + +1. Deploy validators in maintenance window. +2. Roll out Concelier/Excititor images with guard middleware enabled (`AOC_GUARD_ENABLED=true`). +3. Run smoke tests (`stella sources ingest --dry-run` fixtures) before resuming production ingestion. + +--- + +## 2 · Container environment flags + +Add the following environment variables to Concelier/Excititor deployments: + +| Variable | Default | Description | +|----------|---------|-------------| +| `AOC_GUARD_ENABLED` | `true` | Enables `AOCWriteGuard` interception. Set `false` only for controlled rollback. | +| `AOC_ALLOW_SUPERSEDES_RETROFIT` | `false` | Allows temporary supersedes backfill during migration. Remove after cutover. | +| `AOC_METRICS_ENABLED` | `true` | Emits `ingestion_write_total`, `aoc_violation_total`, etc. | +| `AOC_TENANT_HEADER` | `X-Stella-Tenant` | Header name expected from Gateway. | +| `AOC_VERIFIER_USER` | `stella-aoc-verify` | Read-only service user used by UI/CLI verification. | + +Compose snippet: + +```yaml +environment: + - AOC_GUARD_ENABLED=true + - AOC_ALLOW_SUPERSEDES_RETROFIT=false + - AOC_METRICS_ENABLED=true + - AOC_TENANT_HEADER=X-Stella-Tenant + - AOC_VERIFIER_USER=stella-aoc-verify +``` + +Ensure `AOC_VERIFIER_USER` exists in Authority with `aoc:verify` scope and no write permissions. + +--- + +## 3 · Verifier identity + +- Create a dedicated client (`stella-aoc-verify`) via Authority bootstrap: + +```yaml +clients: + - clientId: stella-aoc-verify + grantTypes: [client_credentials] + scopes: [aoc:verify, advisory:verify, vex:verify] + tenants: [default] +``` + +- Store credentials in secret store (`Kubernetes Secret`, `Docker swarm secret`). +- Bind credentials to `stella aoc verify` CI jobs and Console verification service. +- Rotate quarterly; document in `ops/authority-key-rotation.md`. + +--- + +## 4 · Deployment steps + +1. **Pre-checks:** Confirm database backups, alerting in maintenance mode, and staging environment validated. +2. **Apply validators:** Run scripts per § 1.1. +3. **Update manifests:** Inject environment variables (§ 2) and mount guard configuration configmaps. +4. **Redeploy services:** Rolling restart Concelier/Excititor pods. Monitor `ingestion_write_total` for steady throughput. +5. **Seed verifier:** Deploy read-only verifier user and store credentials. +6. **Run verification:** Execute `stella aoc verify --since 24h` and ensure exit code `0`. +7. **Update dashboards:** Point Grafana panels to new metrics (`aoc_violation_total`). +8. **Record handoff:** Capture console screenshots and verification logs for release notes. + +--- + +## 5 · Offline Kit updates + +- Ship validator scripts with Offline Kit (`offline-kit/scripts/apply-aoc-validators.js`). +- Include pre-generated verification reports for air-gapped deployments. +- Document offline CLI workflow in bundle README referencing `docs/cli/cli-reference.md`. +- Ensure `stella-aoc-verify` credentials are scoped to offline tenant and rotated during bundle refresh. + +--- + +## 6 · Rollback plan + +1. Disable guard via `AOC_GUARD_ENABLED=false` on Concelier/Excititor and rollout. +2. Remove validators with the migration script (`--remove`). +3. Pause verification jobs to prevent noise. +4. Investigate and remediate upstream issues before re-enabling guards. + +--- + +## 7 · References + +- [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) +- [Authority scopes & tenancy](../security/authority-scopes.md) +- [Observability guide](../observability/observability.md) +- [CLI AOC commands](../cli/cli-reference.md) +- [Concelier architecture](../ARCHITECTURE_CONCELIER.md) +- [Excititor architecture](../ARCHITECTURE_EXCITITOR.md) + +--- + +## 8 · Compliance checklist + +- [ ] Validators documented and scripts referenced for online/offline deployments. +- [ ] Environment variables cover guard enablement, metrics, and tenant header. +- [ ] Read-only verifier user installation steps included. +- [ ] Offline kit instructions align with validator/verification workflow. +- [ ] Rollback procedure captured. +- [ ] Cross-links to AOC docs, Authority scopes, and observability guides present. +- [ ] DevOps Guild sign-off tracked (owner: @devops-guild, due 2025-10-29). + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/devops/policy-schema-export.md b/docs/devops/policy-schema-export.md new file mode 100644 index 00000000..a0d72a70 --- /dev/null +++ b/docs/devops/policy-schema-export.md @@ -0,0 +1,25 @@ +# Policy Schema Export Automation + +This utility generates JSON Schema documents for the Policy Engine run contracts. + +## Command + +``` +scripts/export-policy-schemas.sh [output-directory] +``` + +When no output directory is supplied, schemas are written to `docs/schemas/`. + +The exporter builds against `StellaOps.Scheduler.Models` and emits: + +- `policy-run-request.schema.json` +- `policy-run-status.schema.json` +- `policy-diff-summary.schema.json` +- `policy-explain-trace.schema.json` + +## CI integration checklist + +- [ ] Invoke the script in the DevOps pipeline (see `DEVOPS-POLICY-20-004`). +- [ ] Publish the generated schemas as pipeline artifacts. +- [ ] Notify downstream consumers when schemas change (Slack `#policy-engine`, changelog snippet). +- [ ] Gate CLI validation once schema artifacts are available. diff --git a/docs/events/README.md b/docs/events/README.md index df0c7625..0583df40 100644 --- a/docs/events/README.md +++ b/docs/events/README.md @@ -2,28 +2,53 @@ Platform services publish strongly typed events; the JSON Schemas in this directory define those envelopes. File names follow `@.json` so producers and consumers can negotiate contracts explicitly. -## Catalog -- `scanner.report.ready@1.json` — emitted by Scanner.WebService once a signed report is persisted (payload embeds the canonical report plus DSSE envelope). Consumers: Notify, UI timeline. -- `scanner.scan.completed@1.json` — emitted alongside the signed report to capture scan outcomes/summary data for downstream automation. Consumers: Notify, Scheduler backfills, UI timelines. -- `scheduler.rescan.delta@1.json` — emitted by Scheduler when BOM-Index diffs require fresh scans. Consumers: Notify, Policy Engine. -- `attestor.logged@1.json` — emitted by Attestor after storing the Rekor inclusion proof. Consumers: UI attestation panel, Governance exports. +## Catalog +**Orchestrator envelopes (ORCH-SVC-38-101)** +- `scanner.event.report.ready@1.json` — orchestrator event emitted when a signed report is persisted. Supersedes the legacy `scanner.report.ready@1` schema and adds versioning, idempotency keys, and trace context. Consumers: Orchestrator bus, Notifications Studio, UI timeline. +- `scanner.event.scan.completed@1.json` — orchestrator event emitted when a scan run finishes. Supersedes the legacy `scanner.scan.completed@1` schema. Consumers: Orchestrator bus, Notifications Studio, Scheduler replay tooling. + +**Legacy envelopes (Redis-backed)** +- `scanner.report.ready@1.json` — legacy Redis stream event emitted once a signed report is persisted (kept for transitional compatibility). +- `scanner.scan.completed@1.json` — legacy Redis stream event emitted alongside the signed report for automation. +- `scheduler.rescan.delta@1.json` — emitted by Scheduler when BOM-Index diffs require fresh scans. Consumers: Notify, Policy Engine. +- `scheduler.graph.job.completed@1.json` — emitted when a Cartographer graph build/overlay job finishes (`status = completed|failed|cancelled`). Consumers: Scheduler WebService (lag metrics/API), Cartographer cache warmers, UI overlay freshness indicators. +- `attestor.logged@1.json` — emitted by Attestor after storing the Rekor inclusion proof. Consumers: UI attestation panel, Governance exports. -Additive payload changes (new optional fields) can stay within the same version. Any breaking change (removing a field, tightening validation, altering semantics) must increment the `@` suffix and update downstream consumers. +Additive payload changes (new optional fields) can stay within the same version. Any breaking change (removing a field, tightening validation, altering semantics) must increment the `@` suffix and update downstream consumers. For full orchestrator guidance see [`orchestrator-scanner-events.md`](orchestrator-scanner-events.md). -## Envelope structure -All event envelopes share the same deterministic header. Use the following table as the quick reference when emitting or parsing events: - -| Field | Type | Notes | -|-------|------|-------| -| `eventId` | `uuid` | Must be globally unique per occurrence; producers log duplicates as fatal. | -| `kind` | `string` | Fixed per schema (e.g., `scanner.report.ready`). Downstream services reject unknown kinds or versions. | -| `tenant` | `string` | Multi‑tenant isolation key; mirror the value recorded in queue/Mongo metadata. | -| `ts` | `date-time` | RFC 3339 UTC timestamp. Use monotonic clocks or atomic offsets so ordering survives retries. | -| `scope` | `object` | Optional block used when the event concerns a specific image or repository. See schema for required fields (e.g., `repo`, `digest`). | -| `payload` | `object` | Event-specific body. Schemas allow additional properties so producers can add optional hints (e.g., `reportId`, `quietedFindingCount`) without breaking consumers. For scanner events, payloads embed both the canonical report document and the DSSE envelope so consumers can reuse signatures without recomputing them. See `docs/runtime/SCANNER_RUNTIME_READINESS.md` for the runtime consumer checklist covering these hints. | +## Envelope structure + +### Orchestrator envelope (version 1) +| Field | Type | Notes | +|-------|------|-------| +| `eventId` | `uuid` | Globally unique per occurrence. | +| `kind` | `string` | e.g., `scanner.event.report.ready`. | +| `version` | `integer` | Schema version (`1` for the initial release). | +| `tenant` | `string` | Multi‑tenant isolation key; mirror the value recorded in queue/Mongo metadata. | +| `occurredAt` | `date-time` | RFC 3339 UTC timestamp describing when the state transition happened. | +| `recordedAt` | `date-time` | RFC 3339 UTC timestamp for durable persistence (optional but recommended). | +| `source` | `string` | Producer identifier (`scanner.webservice`). | +| `idempotencyKey` | `string` | Deterministic dedupe key (`scanner.event.*::`). | +| `correlationId` | `string` | Ties the event to the originating scan/API request. | +| `traceId` / `spanId` | `string` | W3C trace context propagated into downstream telemetry. | +| `scope` | `object` | Optional block with at least `repo` and `digest`. | +| `payload` | `object` | Event-specific body; schemas embed the canonical report and DSSE envelope. | +| `attributes` | `object` | Optional metadata bag (`string` keys/values) for downstream correlation. | + +For Scanner orchestrator events, `links` include console and API deep links (`ui`, `report`, and `policy`) plus an optional `attestation` URL when a DSSE envelope is present. See [`orchestrator-scanner-events.md`](orchestrator-scanner-events.md) for details. + +### Legacy Redis envelope +| Field | Type | Notes | +|-------|------|-------| +| `eventId` | `uuid` | Must be globally unique per occurrence; producers log duplicates as fatal. | +| `kind` | `string` | Fixed per schema (e.g., `scanner.report.ready`). Downstream services reject unknown kinds or versions. | +| `tenant` | `string` | Multi‑tenant isolation key; mirror the value recorded in queue/Mongo metadata. | +| `ts` | `date-time` | RFC 3339 UTC timestamp. Use monotonic clocks or atomic offsets so ordering survives retries. | +| `scope` | `object` | Optional block used when the event concerns a specific image or repository. See schema for required fields (e.g., `repo`, `digest`). | +| `payload` | `object` | Event-specific body. Schemas allow additional properties so producers can add optional hints (e.g., `reportId`, `quietedFindingCount`) without breaking consumers. See `docs/runtime/SCANNER_RUNTIME_READINESS.md` for the runtime consumer checklist covering these hints. | | `attributes` | `object` | Optional metadata bag (`string` keys/values) for downstream correlation (e.g., pipeline identifiers). Omit when unused to keep payloads concise. | - -When adding new optional fields, document the behaviour in the schema’s `description` block and update the consumer checklist in the next sprint sync. + +When adding new optional fields, document the behaviour in the schema’s `description` block and update the consumer checklist in the next sprint sync. ## Canonical samples & validation Reference payloads live under `docs/events/samples/`, mirroring the schema version (`@.sample.json`). They illustrate common field combinations, including the optional attributes that downstream teams rely on for UI affordances and audit trails. Scanner samples reuse the exact DSSE envelope checked into `samples/api/reports/report-sample.dsse.json`, and unit tests (`ReportSamplesTests`, `PlatformEventSchemaValidationTests`) guard that payloads stay canonical and continue to satisfy the published schemas. diff --git a/docs/events/orchestrator-scanner-events.md b/docs/events/orchestrator-scanner-events.md new file mode 100644 index 00000000..cbdb5950 --- /dev/null +++ b/docs/events/orchestrator-scanner-events.md @@ -0,0 +1,121 @@ +# Scanner Orchestrator Events (ORCH-SVC-38-101) + +Last updated: 2025-10-26 + +The Notifications Studio initiative (NOTIFY-SVC-38-001) and orchestrator backlog (ORCH-SVC-38-101) standardise how platform services emit lifecycle events. This document describes the Scanner WebService contract for the new **orchestrator envelopes** (`scanner.event.*`) and how they supersede the legacy Redis-backed `scanner.report.ready` / `scanner.scan.completed` events. + +## 1. Envelope overview + +Orchestrator events share a deterministic JSON envelope: + +| Field | Type | Notes | +|-------|------|-------| +| `eventId` | `uuid` | Globally unique identifier generated per occurrence. | +| `kind` | `string` | Event identifier; Scanner emits `scanner.event.report.ready` and `scanner.event.scan.completed`. | +| `version` | `integer` | Schema version. Initial release uses `1`. | +| `tenant` | `string` | Tenant that owns the scan/report. Mirrors Authority claims. | +| `occurredAt` | `date-time` | UTC instant when the underlying state transition happened (e.g., report persisted). | +| `recordedAt` | `date-time` | UTC instant when the event was durably written. Optional but recommended. | +| `source` | `string` | Producer identifier (`scanner.webservice`). | +| `idempotencyKey` | `string` | Deterministic key for duplicate suppression (see §4). | +| `correlationId` | `string` | Maps back to the API request or scan identifier. | +| `traceId` / `spanId` | `string` | W3C trace context propagated into downstream telemetry. | +| `scope` | `object` | Describes the affected artefact. Requires `repo` and `digest`; optional `namespace`, `component`, `image`. | +| `attributes` | `object` | Flat string map for frequently queried metadata (e.g., policy revision). | +| `payload` | `object` | Event-specific body (see §2). | + +Canonical schemas live under `docs/events/scanner.event.*@1.json`. Samples that round-trip through `NotifyCanonicalJsonSerializer` are stored in `docs/events/samples/`. + +## 2. Event kinds and payloads + +### 2.1 `scanner.event.report.ready` + +Emitted once a signed report is persisted and attested. Payload highlights: + +- `reportId` / `scanId` — identifiers for the persisted report and originating scan. Until Scan IDs are surfaced by the API, `scanId` mirrors `reportId` so downstream correlators can stabilise on a single key. +- **Attributes:** `reportId`, `policyRevisionId`, `policyDigest`, `verdict` — pre-sorted for deterministic routing. +- **Links:** + - `ui` → `/ui/reports/{reportId}` on the current host. + - `report` → `{apiBasePath}/{reportsSegment}/{reportId}` (defaults to `/api/v1/reports/{reportId}`). + - `policy` → `{apiBasePath}/{policySegment}/revisions/{revisionId}` when a revision is present. + - `attestation` → `/ui/attestations/{reportId}` when a DSSE envelope is included. +- `imageDigest` — OCI image digest associated with the analysis. +- `generatedAt` — report generation timestamp (ISO-8601 UTC). +- `verdict` — `pass`, `warn`, or `fail` after policy evaluation. +- `summary` — blocked/warned/ignored/quieted counters (all non-negative integers). +- `delta` — newly critical/high counts and optional `kev` array. +- `quietedFindingCount` — mirrors `summary.quieted`. +- `policy` — revision metadata (`digest`, `revisionId`) surfaced for routing. +- `links` — UI/report/policy URLs suitable for operators. +- `dsse` — embedded DSSE envelope (payload, type, signature list). +- `report` — canonical report document; identical to the DSSE payload. + +Schema: `docs/events/scanner.event.report.ready@1.json` +Sample: `docs/events/samples/scanner.event.report.ready@1.sample.json` + +### 2.2 `scanner.event.scan.completed` + +Emitted after scan execution finishes (success or policy failure). Payload highlights: + +- `reportId` / `scanId` / `imageDigest` — identifiers mirroring the report-ready event. As with the report-ready payload, `scanId` currently mirrors `reportId` as a temporary shim. +- **Attributes:** `reportId`, `policyRevisionId`, `policyDigest`, `verdict`. +- **Links:** same as above (`ui`, `report`, `policy`) with `attestation` populated when DSSE metadata exists. +- `verdict`, `summary`, `delta`, `policy` — same semantics as above. +- `findings` — array of surfaced findings with `id`, `severity`, optional `cve`, `purl`, and `reachability`. +- `links`, `dsse`, `report` — same structure as §2.1 (allows Notifier to reuse signatures). + +Schema: `docs/events/scanner.event.scan.completed@1.json` +Sample: `docs/events/samples/scanner.event.scan.completed@1.sample.json` + +### 2.3 Relationship to legacy events + +| Legacy Redis event | Replacement orchestrator event | Notes | +|--------------------|-------------------------------|-------| +| `scanner.report.ready` | `scanner.event.report.ready` | Adds versioning, idempotency, trace context. Payload is a superset of the legacy fields. | +| `scanner.scan.completed` | `scanner.event.scan.completed` | Same data plus explicit scan identifiers and orchestrator metadata. | + +Legacy schemas remain for backwards-compatibility during migration, but new integrations **must** target the orchestrator variants. + +## 3. Deterministic serialization + +- Producers must serialise events using `NotifyCanonicalJsonSerializer` to guarantee consistent key ordering and whitespace. +- Timestamps (`occurredAt`, `recordedAt`, `payload.generatedAt`) use `DateTimeOffset.UtcDateTime.ToString("O")`. +- Payload arrays (`delta.kev`, `findings`) should be pre-sorted (e.g., alphabetical CVE order) so hash-based consumers remain stable. +- Optional fields are omitted rather than emitted as `null`. + +## 4. Idempotency and correlation + +Idempotency keys dedupe repeated publishes and align with the orchestrator’s outbox pattern: + +| Event kind | Idempotency key template | +|------------|-------------------------| +| `scanner.event.report.ready` | `scanner.event.report.ready::` | +| `scanner.event.scan.completed` | `scanner.event.scan.completed::` | + +Keys are ASCII lowercase; components should be trimmed and validated before concatenation. Retries must reuse the same key. + +`correlationId` should match the scan identifier that appears in REST responses (`scanId`). Re-using the same value across the pair of events allows Notifier and orchestrator analytics to stitch lifecycle data together. + +## 5. Versioning and evolution + +- Increment the `version` field and the `@` suffix for **breaking** changes (field removals, type changes, semantic shifts). +- Additive optional fields may remain within version 1; update the JSON schema and samples accordingly. +- When introducing `@2`, keep the `@1` schema/docs in place until orchestrator subscribers confirm migration. + +## 6. Consumer checklist + +1. Validate incoming payloads against the schema for the targeted version. +2. Use `idempotencyKey` for dedupe, not `eventId`. +3. Map `traceId`/`spanId` into telemetry spans to preserve causality. +4. Prefer `payload.report` → `policy.revisionId` when populating templates; the top-level `attributes` are convenience duplicates for quick routing. +5. Reserve the legacy Redis events for transitional compatibility only; downstream systems should subscribe to the orchestrator bus exposed by ORCH-SVC-38-101. + +## 7. Implementation status and next actions + +- **Scanner WebService** — `SCANNER-EVENTS-16-301` (blocked) and `SCANNER-EVENTS-16-302` (doing) track the production of these envelopes. The remaining blocker is the .NET 10 preview OpenAPI/Auth dependency drift that currently breaks `dotnet test`. Once Gateway and Notifier owners land the replacement packages, rerun the full test suite and capture fresh fixtures under `docs/events/samples/`. +- **Gateway/Notifier consumers** — subscribe to the orchestrator stream documented in ORCH-SVC-38-101. When the Scanner tasks unblock, regenerate notifier contract tests against the sample events included here. +- **Docs cadence** — update this file and the matching JSON schemas whenever payload fields change. Use the rehearsal checklist in `docs/ops/launch-cutover.md` to confirm downstream validation before the production cutover. Record gaps or newly required fields in `docs/ops/launch-readiness.md` so they land in the launch checklist. + +--- + +**Imposed rule reminder:** work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. diff --git a/docs/events/samples/scanner.event.report.ready@1.sample.json b/docs/events/samples/scanner.event.report.ready@1.sample.json new file mode 100644 index 00000000..bcfb8742 --- /dev/null +++ b/docs/events/samples/scanner.event.report.ready@1.sample.json @@ -0,0 +1,93 @@ +{ + "eventId": "6d2d1b77-f3c3-4f70-8a9d-6f2d0c8801ab", + "kind": "scanner.event.report.ready", + "version": 1, + "tenant": "tenant-alpha", + "occurredAt": "2025-10-19T12:34:56Z", + "recordedAt": "2025-10-19T12:34:57Z", + "source": "scanner.webservice", + "idempotencyKey": "scanner.event.report.ready:tenant-alpha:report-abc", + "correlationId": "report-abc", + "traceId": "0af7651916cd43dd8448eb211c80319c", + "spanId": "b7ad6b7169203331", + "scope": { + "namespace": "acme/edge", + "repo": "api", + "digest": "sha256:feedface" + }, + "attributes": { + "reportId": "report-abc", + "policyRevisionId": "rev-42", + "policyDigest": "digest-123", + "verdict": "blocked" + }, + "payload": { + "reportId": "report-abc", + "scanId": "report-abc", + "imageDigest": "sha256:feedface", + "generatedAt": "2025-10-19T12:34:56Z", + "verdict": "fail", + "summary": { + "total": 1, + "blocked": 1, + "warned": 0, + "ignored": 0, + "quieted": 0 + }, + "delta": { + "newCritical": 1, + "kev": [ + "CVE-2024-9999" + ] + }, + "quietedFindingCount": 0, + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "links": { + "ui": "https://scanner.example/ui/reports/report-abc", + "report": "https://scanner.example/api/v1/reports/report-abc", + "policy": "https://scanner.example/api/v1/policy/revisions/rev-42", + "attestation": "https://scanner.example/ui/attestations/report-abc" + }, + "dsse": { + "payloadType": "application/vnd.stellaops.report+json", + "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC1hYmMiLCJpbWFnZURpZ2VzdCI6InNoYTI1NjpmZWVkZmFjZSIsImdlbmVyYXRlZEF0IjoiMjAyNS0xMC0xOVQxMjozNDo1NiswMDowMCIsInZlcmRpY3QiOiJibG9ja2VkIiwicG9saWN5Ijp7InJldmlzaW9uSWQiOiJyZXYtNDIiLCJkaWdlc3QiOiJkaWdlc3QtMTIzIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwic2NvcmUiOjQ3LjUsInNvdXJjZVRydXN0IjoiTlZEIiwicmVhY2hhYmlsaXR5IjoicnVudGltZSJ9XSwiaXNzdWVzIjpbXX0=", + "signatures": [ + { + "keyId": "test-key", + "algorithm": "hs256", + "signature": "signature-value" + } + ] + }, + "report": { + "reportId": "report-abc", + "generatedAt": "2025-10-19T12:34:56Z", + "imageDigest": "sha256:feedface", + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "summary": { + "total": 1, + "blocked": 1, + "warned": 0, + "ignored": 0, + "quieted": 0 + }, + "verdict": "blocked", + "verdicts": [ + { + "findingId": "finding-1", + "status": "Blocked", + "score": 47.5, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ], + "issues": [] + } + } +} diff --git a/docs/events/samples/scanner.event.scan.completed@1.sample.json b/docs/events/samples/scanner.event.scan.completed@1.sample.json new file mode 100644 index 00000000..f8c2634e --- /dev/null +++ b/docs/events/samples/scanner.event.scan.completed@1.sample.json @@ -0,0 +1,99 @@ +{ + "eventId": "08a6de24-4a94-4d14-8432-9d14f36f6da3", + "kind": "scanner.event.scan.completed", + "version": 1, + "tenant": "tenant-alpha", + "occurredAt": "2025-10-19T12:34:56Z", + "recordedAt": "2025-10-19T12:34:57Z", + "source": "scanner.webservice", + "idempotencyKey": "scanner.event.scan.completed:tenant-alpha:report-abc", + "correlationId": "report-abc", + "traceId": "4bf92f3577b34da6a3ce929d0e0e4736", + "scope": { + "namespace": "acme/edge", + "repo": "api", + "digest": "sha256:feedface" + }, + "attributes": { + "reportId": "report-abc", + "policyRevisionId": "rev-42", + "policyDigest": "digest-123", + "verdict": "blocked" + }, + "payload": { + "reportId": "report-abc", + "scanId": "report-abc", + "imageDigest": "sha256:feedface", + "verdict": "fail", + "summary": { + "total": 1, + "blocked": 1, + "warned": 0, + "ignored": 0, + "quieted": 0 + }, + "delta": { + "newCritical": 1, + "kev": [ + "CVE-2024-9999" + ] + }, + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "findings": [ + { + "id": "finding-1", + "severity": "Critical", + "cve": "CVE-2024-9999", + "purl": "pkg:docker/acme/edge-api@sha256-feedface", + "reachability": "runtime" + } + ], + "links": { + "ui": "https://scanner.example/ui/reports/report-abc", + "report": "https://scanner.example/api/v1/reports/report-abc", + "policy": "https://scanner.example/api/v1/policy/revisions/rev-42", + "attestation": "https://scanner.example/ui/attestations/report-abc" + }, + "dsse": { + "payloadType": "application/vnd.stellaops.report+json", + "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC1hYmMiLCJpbWFnZURpZ2VzdCI6InNoYTI1NjpmZWVkZmFjZSIsImdlbmVyYXRlZEF0IjoiMjAyNS0xMC0xOVQxMjozNDo1NiswMDowMCIsInZlcmRpY3QiOiJibG9ja2VkIiwicG9saWN5Ijp7InJldmlzaW9uSWQiOiJyZXYtNDIiLCJkaWdlc3QiOiJkaWdlc3QtMTIzIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwic2NvcmUiOjQ3LjUsInNvdXJjZVRydXN0IjoiTlZEIiwicmVhY2hhYmlsaXR5IjoicnVudGltZSJ9XSwiaXNzdWVzIjpbXX0=", + "signatures": [ + { + "keyId": "test-key", + "algorithm": "hs256", + "signature": "signature-value" + } + ] + }, + "report": { + "reportId": "report-abc", + "generatedAt": "2025-10-19T12:34:56Z", + "imageDigest": "sha256:feedface", + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "summary": { + "total": 1, + "blocked": 1, + "warned": 0, + "ignored": 0, + "quieted": 0 + }, + "verdict": "blocked", + "verdicts": [ + { + "findingId": "finding-1", + "status": "Blocked", + "score": 47.5, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ], + "issues": [] + } + } +} diff --git a/docs/events/samples/scheduler.graph.job.completed@1.sample.json b/docs/events/samples/scheduler.graph.job.completed@1.sample.json new file mode 100644 index 00000000..6dfd91d0 --- /dev/null +++ b/docs/events/samples/scheduler.graph.job.completed@1.sample.json @@ -0,0 +1,36 @@ +{ + "eventId": "4d33c19c-1c8a-44d1-9954-1d5e98b2af71", + "kind": "scheduler.graph.job.completed", + "tenant": "tenant-alpha", + "ts": "2025-10-26T12:00:45Z", + "payload": { + "jobType": "build", + "status": "completed", + "occurredAt": "2025-10-26T12:00:45Z", + "job": { + "schemaVersion": "scheduler.graph-build-job@1", + "id": "gbj_20251026a", + "tenantId": "tenant-alpha", + "sbomId": "sbom_20251026", + "sbomVersionId": "sbom_ver_20251026", + "sbomDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "graphSnapshotId": "graph_snap_20251026", + "status": "completed", + "trigger": "sbom-version", + "attempts": 1, + "cartographerJobId": "carto_job_42", + "correlationId": "evt_svc_987", + "createdAt": "2025-10-26T12:00:00+00:00", + "startedAt": "2025-10-26T12:00:05+00:00", + "completedAt": "2025-10-26T12:00:45+00:00", + "metadata": { + "sbomEventId": "sbom_evt_20251026" + } + }, + "resultUri": "oras://cartographer/offline/tenant-alpha/graph_snap_20251026" + }, + "attributes": { + "cartographerCluster": "offline-kit", + "plannerShard": "graph-builders-01" + } +} diff --git a/docs/events/scanner.event.report.ready@1.json b/docs/events/scanner.event.report.ready@1.json new file mode 100644 index 00000000..5d78d68c --- /dev/null +++ b/docs/events/scanner.event.report.ready@1.json @@ -0,0 +1,164 @@ +{ + "$id": "https://stella-ops.org/schemas/events/scanner.event.report.ready@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Scanner orchestrator event – report ready (v1)", + "type": "object", + "additionalProperties": false, + "required": [ + "eventId", + "kind", + "version", + "tenant", + "occurredAt", + "source", + "idempotencyKey", + "payload" + ], + "properties": { + "eventId": { + "type": "string", + "format": "uuid", + "description": "Globally unique identifier for this occurrence." + }, + "kind": { + "const": "scanner.event.report.ready", + "description": "Event kind identifier consumed by orchestrator subscribers." + }, + "version": { + "const": 1, + "description": "Schema version for orchestrator envelopes." + }, + "tenant": { + "type": "string", + "description": "Tenant that owns the scan/report." + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "description": "Timestamp (UTC) when the report transitioned to ready." + }, + "recordedAt": { + "type": "string", + "format": "date-time", + "description": "Timestamp (UTC) when the event was persisted. Optional." + }, + "source": { + "type": "string", + "description": "Producer identifier, e.g. `scanner.webservice`." + }, + "idempotencyKey": { + "type": "string", + "minLength": 8, + "description": "Deterministic key used to deduplicate events downstream." + }, + "correlationId": { + "type": "string", + "description": "Correlation identifier that ties this event to a request or workflow." + }, + "traceId": { + "type": "string", + "description": "W3C trace ID (32 hex chars) for distributed tracing." + }, + "spanId": { + "type": "string", + "description": "Optional span identifier associated with traceId." + }, + "scope": { + "type": "object", + "additionalProperties": false, + "required": ["repo", "digest"], + "properties": { + "namespace": {"type": "string"}, + "repo": {"type": "string"}, + "digest": {"type": "string"}, + "component": {"type": "string"}, + "image": {"type": "string"} + } + }, + "attributes": { + "type": "object", + "description": "String attributes for downstream correlation (policy revision, scan id, etc.).", + "additionalProperties": {"type": "string"} + }, + "payload": { + "type": "object", + "additionalProperties": true, + "required": ["reportId", "verdict", "summary", "links", "report"], + "properties": { + "reportId": {"type": "string"}, + "scanId": {"type": "string"}, + "imageDigest": {"type": "string"}, + "generatedAt": {"type": "string", "format": "date-time"}, + "verdict": {"enum": ["pass", "warn", "fail"]}, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["total", "blocked", "warned", "ignored", "quieted"], + "properties": { + "total": {"type": "integer", "minimum": 0}, + "blocked": {"type": "integer", "minimum": 0}, + "warned": {"type": "integer", "minimum": 0}, + "ignored": {"type": "integer", "minimum": 0}, + "quieted": {"type": "integer", "minimum": 0} + } + }, + "delta": { + "type": "object", + "additionalProperties": false, + "properties": { + "newCritical": {"type": "integer", "minimum": 0}, + "newHigh": {"type": "integer", "minimum": 0}, + "kev": { + "type": "array", + "items": {"type": "string"} + } + } + }, + "quietedFindingCount": { + "type": "integer", + "minimum": 0 + }, + "policy": { + "type": "object", + "description": "Policy revision metadata surfaced alongside the report." + }, + "links": { + "type": "object", + "additionalProperties": false, + "properties": { + "ui": {"type": "string", "format": "uri"}, + "report": {"type": "string", "format": "uri"}, + "policy": {"type": "string", "format": "uri"}, + "attestation": {"type": "string", "format": "uri"} + } + }, + "dsse": { + "type": "object", + "additionalProperties": false, + "required": ["payloadType", "payload", "signatures"], + "properties": { + "payloadType": {"type": "string"}, + "payload": {"type": "string"}, + "signatures": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["keyId", "algorithm", "signature"], + "properties": { + "keyId": {"type": "string"}, + "algorithm": {"type": "string"}, + "signature": {"type": "string"} + } + } + } + } + }, + "report": { + "type": "object", + "description": "Canonical scanner report document that aligns with the DSSE payload." + } + } + } + } +} diff --git a/docs/events/scanner.event.scan.completed@1.json b/docs/events/scanner.event.scan.completed@1.json new file mode 100644 index 00000000..d89ffcfc --- /dev/null +++ b/docs/events/scanner.event.scan.completed@1.json @@ -0,0 +1,174 @@ +{ + "$id": "https://stella-ops.org/schemas/events/scanner.event.scan.completed@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Scanner orchestrator event – scan completed (v1)", + "type": "object", + "additionalProperties": false, + "required": [ + "eventId", + "kind", + "version", + "tenant", + "occurredAt", + "source", + "idempotencyKey", + "payload" + ], + "properties": { + "eventId": { + "type": "string", + "format": "uuid", + "description": "Globally unique identifier for this occurrence." + }, + "kind": { + "const": "scanner.event.scan.completed", + "description": "Event kind identifier consumed by orchestrator subscribers." + }, + "version": { + "const": 1, + "description": "Schema version for orchestrator envelopes." + }, + "tenant": { + "type": "string", + "description": "Tenant that owns the scan." + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "description": "Timestamp (UTC) when the scan completed." + }, + "recordedAt": { + "type": "string", + "format": "date-time", + "description": "Timestamp (UTC) when the event was persisted. Optional." + }, + "source": { + "type": "string", + "description": "Producer identifier, e.g. `scanner.webservice`." + }, + "idempotencyKey": { + "type": "string", + "minLength": 8, + "description": "Deterministic key used to deduplicate events downstream." + }, + "correlationId": { + "type": "string", + "description": "Correlation identifier tying this event to a request or workflow." + }, + "traceId": { + "type": "string", + "description": "W3C trace ID (32 hex chars) for distributed tracing." + }, + "spanId": { + "type": "string", + "description": "Optional span identifier associated with traceId." + }, + "scope": { + "type": "object", + "additionalProperties": false, + "required": ["repo", "digest"], + "properties": { + "namespace": {"type": "string"}, + "repo": {"type": "string"}, + "digest": {"type": "string"}, + "component": {"type": "string"}, + "image": {"type": "string"} + } + }, + "attributes": { + "type": "object", + "description": "String attributes for downstream correlation (policy revision, scan id, etc.).", + "additionalProperties": {"type": "string"} + }, + "payload": { + "type": "object", + "additionalProperties": true, + "required": ["reportId", "scanId", "imageDigest", "verdict", "summary", "report"], + "properties": { + "reportId": {"type": "string"}, + "scanId": {"type": "string"}, + "imageDigest": {"type": "string"}, + "verdict": {"enum": ["pass", "warn", "fail"]}, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["total", "blocked", "warned", "ignored", "quieted"], + "properties": { + "total": {"type": "integer", "minimum": 0}, + "blocked": {"type": "integer", "minimum": 0}, + "warned": {"type": "integer", "minimum": 0}, + "ignored": {"type": "integer", "minimum": 0}, + "quieted": {"type": "integer", "minimum": 0} + } + }, + "delta": { + "type": "object", + "additionalProperties": false, + "properties": { + "newCritical": {"type": "integer", "minimum": 0}, + "newHigh": {"type": "integer", "minimum": 0}, + "kev": { + "type": "array", + "items": {"type": "string"} + } + } + }, + "policy": { + "type": "object", + "description": "Policy revision metadata surfaced alongside the report." + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": {"type": "string"}, + "severity": {"type": "string"}, + "cve": {"type": "string"}, + "purl": {"type": "string"}, + "reachability": {"type": "string"} + } + } + }, + "links": { + "type": "object", + "additionalProperties": false, + "properties": { + "ui": {"type": "string", "format": "uri"}, + "report": {"type": "string", "format": "uri"}, + "policy": {"type": "string", "format": "uri"}, + "attestation": {"type": "string", "format": "uri"} + } + }, + "dsse": { + "type": "object", + "additionalProperties": false, + "required": ["payloadType", "payload", "signatures"], + "properties": { + "payloadType": {"type": "string"}, + "payload": {"type": "string"}, + "signatures": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["keyId", "algorithm", "signature"], + "properties": { + "keyId": {"type": "string"}, + "algorithm": {"type": "string"}, + "signature": {"type": "string"} + } + } + } + } + }, + "report": { + "type": "object", + "description": "Canonical scanner report document that aligns with the DSSE payload." + } + } + } + } +} diff --git a/docs/events/scheduler.graph.job.completed@1.json b/docs/events/scheduler.graph.job.completed@1.json new file mode 100644 index 00000000..03b519c9 --- /dev/null +++ b/docs/events/scheduler.graph.job.completed@1.json @@ -0,0 +1,196 @@ +{ + "$id": "https://stella-ops.org/schemas/events/scheduler.graph.job.completed@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Scheduler Graph Job Completed Event", + "description": "Legacy scheduler event emitted when a graph build or overlay job reaches a terminal state. Consumers validate downstream caches and surface overlay freshness.", + "type": "object", + "additionalProperties": false, + "required": ["eventId", "kind", "tenant", "ts", "payload"], + "properties": { + "eventId": { + "type": "string", + "format": "uuid", + "description": "Globally unique identifier per event." + }, + "kind": { + "const": "scheduler.graph.job.completed" + }, + "tenant": { + "type": "string", + "description": "Tenant identifier scoped to the originating job." + }, + "ts": { + "type": "string", + "format": "date-time", + "description": "UTC timestamp when the job reached a terminal state." + }, + "payload": { + "type": "object", + "additionalProperties": false, + "required": ["jobType", "job", "status", "occurredAt"], + "properties": { + "jobType": { + "type": "string", + "enum": ["build", "overlay"], + "description": "Job flavour, matches the CLR type of the serialized job payload." + }, + "status": { + "type": "string", + "enum": ["completed", "failed", "cancelled"], + "description": "Terminal status recorded for the job." + }, + "occurredAt": { + "type": "string", + "format": "date-time", + "description": "UTC timestamp of the terminal transition, mirrors job.CompletedAt." + }, + "job": { + "oneOf": [ + {"$ref": "#/definitions/graphBuildJob"}, + {"$ref": "#/definitions/graphOverlayJob"} + ], + "description": "Canonical serialized representation of the finished job." + }, + "resultUri": { + "type": "string", + "description": "Optional URI pointing to Cartographer snapshot or overlay bundle (if available)." + } + } + }, + "attributes": { + "type": "object", + "description": "Optional correlation bag for downstream consumers.", + "additionalProperties": { + "type": "string" + } + } + }, + "definitions": { + "graphBuildJob": { + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "id", + "tenantId", + "sbomId", + "sbomVersionId", + "sbomDigest", + "status", + "trigger", + "attempts", + "createdAt" + ], + "properties": { + "schemaVersion": { + "const": "scheduler.graph-build-job@1" + }, + "id": {"type": "string"}, + "tenantId": {"type": "string"}, + "sbomId": {"type": "string"}, + "sbomVersionId": {"type": "string"}, + "sbomDigest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$" + }, + "graphSnapshotId": {"type": "string"}, + "status": { + "type": "string", + "enum": ["pending", "queued", "running", "completed", "failed", "cancelled"] + }, + "trigger": { + "type": "string", + "enum": ["sbom-version", "backfill", "manual"] + }, + "attempts": { + "type": "integer", + "minimum": 0 + }, + "cartographerJobId": {"type": "string"}, + "correlationId": {"type": "string"}, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "startedAt": { + "type": "string", + "format": "date-time" + }, + "completedAt": { + "type": "string", + "format": "date-time" + }, + "error": {"type": "string"}, + "metadata": { + "type": "object", + "additionalProperties": {"type": "string"} + } + } + }, + "graphOverlayJob": { + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "id", + "tenantId", + "graphSnapshotId", + "overlayKind", + "overlayKey", + "status", + "trigger", + "attempts", + "createdAt" + ], + "properties": { + "schemaVersion": { + "const": "scheduler.graph-overlay-job@1" + }, + "id": {"type": "string"}, + "tenantId": {"type": "string"}, + "graphSnapshotId": {"type": "string"}, + "buildJobId": {"type": "string"}, + "overlayKind": { + "type": "string", + "enum": ["policy", "advisory", "vex"] + }, + "overlayKey": {"type": "string"}, + "subjects": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + "status": { + "type": "string", + "enum": ["pending", "queued", "running", "completed", "failed", "cancelled"] + }, + "trigger": { + "type": "string", + "enum": ["policy", "advisory", "vex", "sbom-version", "manual"] + }, + "attempts": { + "type": "integer", + "minimum": 0 + }, + "correlationId": {"type": "string"}, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "startedAt": { + "type": "string", + "format": "date-time" + }, + "completedAt": { + "type": "string", + "format": "date-time" + }, + "error": {"type": "string"}, + "metadata": { + "type": "object", + "additionalProperties": {"type": "string"} + } + } + } + } +} diff --git a/docs/examples/policies/README.md b/docs/examples/policies/README.md new file mode 100644 index 00000000..c46ede0e --- /dev/null +++ b/docs/examples/policies/README.md @@ -0,0 +1,16 @@ +# Policy Examples + +Sample `stella-dsl@1` policies illustrating common deployment personas. Each example includes commentary, CLI usage hints, and a compliance checklist. + +| Example | Description | +|---------|-------------| +| [Baseline](baseline.md) | Balanced production defaults (block critical, respect strong VEX). | +| [Serverless](serverless.md) | Aggressive blocking for serverless workloads (no High+, pinned base images). | +| [Internal Only](internal-only.md) | Lenient policy for internal/dev environments with KEV safeguards. | + +Policy source files (`*.stella`) live alongside the documentation so you can copy/paste or use `stella policy new --from file://...`. + +--- + +*Last updated: 2025-10-26.* + diff --git a/docs/examples/policies/baseline.md b/docs/examples/policies/baseline.md new file mode 100644 index 00000000..76c0eb0d --- /dev/null +++ b/docs/examples/policies/baseline.md @@ -0,0 +1,79 @@ +# Baseline Policy Example (`baseline.stella`) + +This sample policy provides a balanced default for production workloads: block critical findings, require strong VEX justifications to suppress advisories, and warn on deprecated runtimes. Use it as a starting point for tenants that want guardrails without excessive noise. + +```dsl +policy "Baseline Production Policy" syntax "stella-dsl@1" { + metadata { + description = "Block critical, escalate high, enforce VEX justifications." + tags = ["baseline","production"] + } + + profile severity { + map vendor_weight { + source "GHSA" => +0.5 + source "OSV" => +0.0 + source "VendorX" => -0.2 + } + env exposure_adjustments { + if env.exposure == "internet" then +0.5 + if env.runtime == "legacy" then +0.3 + } + } + + rule block_critical priority 5 { + when severity.normalized >= "Critical" + then status := "blocked" + because "Critical severity must be remediated before deploy." + } + + rule escalate_high_internet { + when severity.normalized == "High" + and env.exposure == "internet" + then escalate to severity_band("Critical") + because "High severity on internet-exposed asset escalates to critical." + } + + rule require_vex_justification { + when vex.any(status in ["not_affected","fixed"]) + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + annotate winning_statement := vex.latest().statementId + because "Respect strong vendor VEX claims." + } + + rule alert_warn_eol_runtime priority 1 { + when severity.normalized <= "Medium" + and sbom.has_tag("runtime:eol") + then warn message "Runtime marked as EOL; upgrade recommended." + because "Deprecated runtime should be upgraded." + } +} +``` + +## Commentary + +- **Severity profile** tightens vendor weights and applies exposure modifiers so internet-facing/high severity pairs escalate automatically. +- **VEX rule** only honours strong justifications, preventing weaker claims from hiding issues. +- **Warnings first** – The `alert_warn_eol_runtime` rule name ensures it sorts before the require-VEX rule, keeping alerts visible without flipping to `RequiresVex`. +- Works well as shared `tenant-global` baseline; use tenant overrides for stricter tolerant environments. + +## Try it out + +```bash +stella policy new --policy-id P-baseline --template blank --open +stella policy lint examples/policies/baseline.stella +stella policy simulate P-baseline --candidate 1 --sbom sbom:sample-prod +``` + +## Compliance checklist + +- [ ] Policy compiled via `stella policy lint` without diagnostics. +- [ ] Simulation diff reviewed against golden SBOM set. +- [ ] Approval note documents rationale before promoting to production. +- [ ] EOL runtime tags kept up to date in SBOM metadata. +- [ ] VEX vendor allow-list reviewed quarterly. + +--- + +*Last updated: 2025-10-26.* diff --git a/docs/examples/policies/baseline.stella b/docs/examples/policies/baseline.stella new file mode 100644 index 00000000..747cab23 --- /dev/null +++ b/docs/examples/policies/baseline.stella @@ -0,0 +1,46 @@ +policy "Baseline Production Policy" syntax "stella-dsl@1" { + metadata { + description = "Block critical, escalate high, enforce VEX justifications." + tags = ["baseline","production"] + } + + profile severity { + map vendor_weight { + source "GHSA" => +0.5 + source "OSV" => +0.0 + source "VendorX" => -0.2 + } + env exposure_adjustments { + if env.exposure == "internet" then +0.5 + if env.runtime == "legacy" then +0.3 + } + } + + rule block_critical priority 5 { + when severity.normalized >= "Critical" + then status := "blocked" + because "Critical severity must be remediated before deploy." + } + + rule escalate_high_internet { + when severity.normalized == "High" + and env.exposure == "internet" + then escalate to severity_band("Critical") + because "High severity on internet-exposed asset escalates to critical." + } + + rule require_vex_justification { + when vex.any(status in ["not_affected","fixed"]) + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + annotate winning_statement := vex.latest().statementId + because "Respect strong vendor VEX claims." + } + + rule alert_warn_eol_runtime priority 1 { + when severity.normalized <= "Medium" + and sbom.has_tag("runtime:eol") + then warn message "Runtime marked as EOL; upgrade recommended." + because "Deprecated runtime should be upgraded." + } +} diff --git a/docs/examples/policies/baseline.yaml b/docs/examples/policies/baseline.yaml new file mode 100644 index 00000000..55940ee6 --- /dev/null +++ b/docs/examples/policies/baseline.yaml @@ -0,0 +1,34 @@ +version: "1.0" +metadata: + description: Baseline production policy + tags: + - baseline + - production +rules: + - name: Block Critical + severity: [Critical] + action: block + + - name: Escalate High Internet + severity: [High] + environments: [internet] + action: + type: escalate + escalate: + minimumSeverity: Critical + + - name: Require VEX justification + sources: [NVD, GHSA] + action: + type: requireVex + requireVex: + vendors: [VendorX, VendorY] + justifications: + - component_not_present + - vulnerable_code_not_present + + - name: Alert warn EOL runtime + priority: 1 + severity: [Low, Medium] + tags: [runtime:eol] + action: warn diff --git a/docs/examples/policies/internal-only.md b/docs/examples/policies/internal-only.md new file mode 100644 index 00000000..60803cc1 --- /dev/null +++ b/docs/examples/policies/internal-only.md @@ -0,0 +1,72 @@ +# Internal-Only Policy Example (`internal-only.stella`) + +A relaxed profile for internal services and development environments: allow Medium severities with warnings, rely on VEX more heavily, but still block KEV/actively exploited advisories. + +```dsl +policy "Internal Only Policy" syntax "stella-dsl@1" { + metadata { + description = "Lenient policy for internal / dev tenants." + tags = ["internal","dev"] + } + + profile severity { + env exposure_adjustments { + if env.exposure == "internal" then -0.4 + if env.stage == "dev" then -0.6 + } + } + + rule block_kev priority 1 { + when advisory.has_tag("kev") + then status := "blocked" + because "Known exploited vulnerabilities must be remediated." + } + + rule allow_medium_with_warning { + when severity.normalized == "Medium" + and env.exposure == "internal" + then warn message "Medium severity permitted in internal environments." + because "Allow Medium findings with warning for internal workloads." + } + + rule accept_vendor_vex { + when vex.any(status in ["not_affected","fixed"]) + then status := vex.status + annotate justification := vex.latest().justification + because "Trust vendor VEX statements for internal scope." + } + + rule quiet_low_priority { + when severity.normalized <= "Low" + then ignore until "2026-01-01T00:00:00Z" + because "Quiet low severity until next annual remediation sweep." + } +} +``` + +## Commentary + +- Suitable for staging/dev tenants with lower blast radius. +- KEV advisories override lenient behaviour to maintain minimum security bar. +- Warnings ensure Medium findings stay visible in dashboards and CLI outputs. +- Quiet rule enforces planned clean-up date; update before expiry. + +## Try it out + +```bash +stella policy lint examples/policies/internal-only.stella +stella policy simulate P-internal --candidate 1 \ + --sbom sbom:internal-service --env exposure=internal --env stage=dev +``` + +## Compliance checklist + +- [ ] Tenant classified as internal-only with documented risk acceptance. +- [ ] KEV feed synced (Concelier) and tags confirmed before relying on rule. +- [ ] Quiet expiry tracked; remediation backlog updated prior to deadline. +- [ ] Developers informed that warnings still affect quality score. +- [ ] Policy not used for production or internet-exposed services. + +--- + +*Last updated: 2025-10-26.* diff --git a/docs/examples/policies/internal-only.stella b/docs/examples/policies/internal-only.stella new file mode 100644 index 00000000..9457c336 --- /dev/null +++ b/docs/examples/policies/internal-only.stella @@ -0,0 +1,39 @@ +policy "Internal Only Policy" syntax "stella-dsl@1" { + metadata { + description = "Lenient policy for internal / dev tenants." + tags = ["internal","dev"] + } + + profile severity { + env exposure_adjustments { + if env.exposure == "internal" then -0.4 + if env.stage == "dev" then -0.6 + } + } + + rule block_kev priority 1 { + when advisory.has_tag("kev") + then status := "blocked" + because "Known exploited vulnerabilities must be remediated." + } + + rule allow_medium_with_warning { + when severity.normalized == "Medium" + and env.exposure == "internal" + then warn message "Medium severity permitted in internal environments." + because "Allow Medium findings with warning for internal workloads." + } + + rule accept_vendor_vex { + when vex.any(status in ["not_affected","fixed"]) + then status := vex.status + annotate justification := vex.latest().justification + because "Trust vendor VEX statements for internal scope." + } + + rule quiet_low_priority { + when severity.normalized <= "Low" + then ignore until "2026-01-01T00:00:00Z" + because "Quiet low severity until next annual remediation sweep." + } +} diff --git a/docs/examples/policies/internal-only.yaml b/docs/examples/policies/internal-only.yaml new file mode 100644 index 00000000..a7106776 --- /dev/null +++ b/docs/examples/policies/internal-only.yaml @@ -0,0 +1,31 @@ +version: "1.0" +metadata: + description: Relaxed internal/development policy + tags: + - internal + - dev +rules: + - name: Block KEV advisories + tags: [kev] + action: block + + - name: Warn medium severity + severity: [Medium] + environments: [internal] + action: warn + + - name: Accept vendor VEX + action: + type: require_vex + requireVex: + vendors: [VendorX, VendorY] + justifications: + - component_not_present + - vulnerable_code_not_present + + - name: Quiet low severity + severity: [Low, Informational] + action: + type: ignore + until: 2026-01-01T00:00:00Z + justification: "Deferred to annual remediation cycle" diff --git a/docs/examples/policies/serverless.md b/docs/examples/policies/serverless.md new file mode 100644 index 00000000..ac14675f --- /dev/null +++ b/docs/examples/policies/serverless.md @@ -0,0 +1,72 @@ +# Serverless Policy Example (`serverless.stella`) + +Optimised for short-lived serverless workloads: focus on runtime integrity, disallow vulnerable layers entirely, and permit temporary suppressions only with strict justification windows. + +```dsl +policy "Serverless Tight Policy" syntax "stella-dsl@1" { + metadata { + description = "Aggressive blocking for serverless runtimes." + tags = ["serverless","prod","strict"] + } + + profile severity { + env runtime_overrides { + if env.runtime == "serverless" then +0.7 + if env.runtime == "batch" then +0.2 + } + } + + rule block_any_high { + when severity.normalized >= "High" + then status := "blocked" + because "Serverless workloads block High+ severities." + } + + rule forbid_unpinned_base { + when sbom.has_tag("image:latest-tag") + then status := "blocked" + because "Base image must be pinned (no :latest)." + } + + rule zero_tolerance_vex { + when vex.any(status == "not_affected") + then requireVex { vendors = ["VendorX","VendorY"], justifications = ["component_not_present"] } + because "Allow not_affected only from trusted vendors with strongest justification." + } + + rule temporary_quiet { + when env.deployment == "canary" + and severity.normalized == "Medium" + then ignore until coalesce(env.quietUntil, "2025-12-31T00:00:00Z") + because "Allow short canary quiet window while fix rolls out." + } +} +``` + +## Commentary + +- Designed for serverless tenants where redeploy cost is low and failing fast is preferred. +- `forbid_unpinned_base` enforces supply-chain best practices. +- `temporary_quiet` ensures quiet windows expire automatically; require deployments to set `env.quietUntil`. +- Intended to be layered on top of baseline (override per tenant) or used standalone for serverless-only accounts. + +## Try it out + +```bash +stella policy lint examples/policies/serverless.stella +stella policy simulate P-serverless --candidate 1 \ + --sbom sbom:lambda-hello --env runtime=serverless --env deployment=canary +``` + +## Compliance checklist + +- [ ] Quiet window expirations tracked and documented. +- [ ] Trusted VEX vendor list reviewed quarterly. +- [ ] Deployment pipeline enforces pinned base images before approval. +- [ ] Canary deployments monitored for recurrence before ignoring Medium severity. +- [ ] Serverless teams acknowledge runbook for blocked deployments. + +--- + +*Last updated: 2025-10-26.* + diff --git a/docs/examples/policies/serverless.stella b/docs/examples/policies/serverless.stella new file mode 100644 index 00000000..6ec91dcb --- /dev/null +++ b/docs/examples/policies/serverless.stella @@ -0,0 +1,39 @@ +policy "Serverless Tight Policy" syntax "stella-dsl@1" { + metadata { + description = "Aggressive blocking for serverless runtimes." + tags = ["serverless","prod","strict"] + } + + profile severity { + env runtime_overrides { + if env.runtime == "serverless" then +0.7 + if env.runtime == "batch" then +0.2 + } + } + + rule block_any_high { + when severity.normalized >= "High" + then status := "blocked" + because "Serverless workloads block High+ severities." + } + + rule forbid_unpinned_base { + when sbom.has_tag("image:latest-tag") + then status := "blocked" + because "Base image must be pinned (no :latest)." + } + + rule zero_tolerance_vex { + when vex.any(status == "not_affected") + then requireVex { vendors = ["VendorX","VendorY"], justifications = ["component_not_present"] } + because "Allow not_affected only from trusted vendors with strongest justification." + } + + rule temporary_quiet { + when env.deployment == "canary" + and severity.normalized == "Medium" + then ignore until coalesce(env.quietUntil, "2025-12-31T00:00:00Z") + because "Allow short canary quiet window while fix rolls out." + } +} + diff --git a/docs/examples/policies/serverless.yaml b/docs/examples/policies/serverless.yaml new file mode 100644 index 00000000..b0db3c98 --- /dev/null +++ b/docs/examples/policies/serverless.yaml @@ -0,0 +1,30 @@ +version: "1.0" +metadata: + description: Strict policy for serverless workloads + tags: + - serverless + - prod + - strict +rules: + - name: Block High And Above + severity: [High, Critical] + action: block + + - name: Forbid Unpinned Base Images + tags: [image:latest-tag] + action: block + + - name: Require Trusted VEX + action: + type: require_vex + requireVex: + vendors: [VendorX, VendorY] + justifications: [component_not_present] + + - name: Quiet Medium Canary + severity: [Medium] + environments: [canary] + action: + type: ignore + until: 2025-12-31T00:00:00Z + justification: "Temporary canary exception" diff --git a/docs/faq/policy-faq.md b/docs/faq/policy-faq.md new file mode 100644 index 00000000..d10e4540 --- /dev/null +++ b/docs/faq/policy-faq.md @@ -0,0 +1,96 @@ +# Policy Engine FAQ + +Answers to questions that Support, Ops, and Policy Guild teams receive most frequently. Pair this FAQ with the [Policy Lifecycle](../policy/lifecycle.md), [Runs](../policy/runs.md), and [CLI guide](../cli/policy.md) for deeper explanations. + +--- + +## Authoring & DSL + +**Q:** *Lint succeeds locally, but submit still fails with `ERR_POL_001`. Why?* +**A:** The CLI requires lint & compile artefacts newer than 24 hours. Re-run `stella policy lint` and `stella policy compile` before submitting; ensure you upload the latest diff files with `--attach`. + +**Q:** *How do I layer tenant-specific overrides on top of the baseline policy?* +**A:** Keep the baseline in `tenant-global`. For tenant overrides, create a policy referencing the baseline via CLI (`stella policy new --from baseline@`), then adjust rules. Activation is per tenant. + +**Q:** *Can I import YAML/Rego policies from earlier releases?* +**A:** No direct import. Use the migration script (`stella policy migrate legacy.yaml`) which outputs `stella-dsl@1` skeletons. Review manually before submission. + +--- + +## Simulation & Determinism + +**Q:** *Simulation shows huge differences even though I only tweaked metadata. What did I miss?* +**A:** Check if your simulation used the same SBOM set/env as previous runs. CLI default uses golden fixtures; UI can store custom presets. Large diffs may also indicate Concelier updates; compare advisory cursors in the Simulation tab. + +**Q:** *How do we guard against non-deterministic behaviour?* +**A:** CI runs `policy simulate` twice with identical inputs and compares outputs (`DEVOPS-POLICY-20-003`). Any difference fails the pipeline. Locally you can use `stella policy run replay` to verify determinism. + +**Q:** *What happens if the determinism guard (`ERR_POL_004`) triggers?* +**A:** Policy Engine halts the run, raises `policy.run.failed` with code `ERR_POL_004`, and switches to incident mode (100 % sampling). Review recent code changes; often caused by new helpers that call `DateTime.Now` or non-allowlisted HTTP clients. + +--- + +## VEX & Suppressions + +**Q:** *A vendor marked a CVE `not_affected` but the policy still blocks. Why?* +**A:** Check the required justifications. Baseline policy only accepts `component_not_present` and `vulnerable_code_not_present`. Other statuses need explicit rules. Use `stella findings explain` to see which VEX statement was considered. + +**Q:** *Can we quiet a finding indefinitely?* +**A:** Avoid indefinite quiets. Policy DSL requires an `until` timestamp. If the use case is permanent, move the rule into baseline logic with strong justification and documentation. + +**Q:** *How do we detect overuse of suppressions?* +**A:** Observability exports `policy_suppressions_total` and CLI `stella policy stats`. Review weekly; Support flags tenants whose suppressions grow faster than remediation tickets. + +--- + +## Runs & Operations + +**Q:** *Incremental runs are backlogged. What should we check first?* +**A:** Inspect `policy_run_queue_depth` and `policy_delta_backlog_age_seconds` dashboards. If queue depth high, scale worker replicas or investigate upstream change storms (Concelier/Excititor). Use `stella policy run list --status failed` for recent errors. + +**Q:** *Full runs take longer than 30 min. Is that a breach?* +**A:** Goal is ≤ 30 min, but large tenants may exceed temporarily. Ensure Mongo indexes are current and that worker nodes meet sizing (4 vCPU). Consider sharding runs by SBOM group. + +**Q:** *How do I replay a run for audit evidence?* +**A:** `stella policy run replay --output replay.tgz` produces a sealed bundle. Upload to evidence locker or attach to incident tickets. + +--- + +## Approvals & Governance + +**Q:** *Can authors approve their own policies?* +**A:** No. Authority denies approval if `approved_by == submitted_by`. Assign at least two reviewers (one security, one product). + +**Q:** *What scopes do bots need for CI pipelines?* +**A:** Typically `policy:read`, `policy:simulate`, `policy:runs`. Only grant `policy:run` if the pipeline should trigger runs. Never give CI tokens `policy:approve`. + +**Q:** *How do we manage policies in air-gapped deployments?* +**A:** Use `stella policy bundle export --sealed` on a connected site, transfer via approved media, then `stella policy bundle import` inside the enclave. Enable `--sealed` flag in CLI/UI to block accidental outbound calls. + +--- + +## Troubleshooting + +**Q:** *API calls return `403` despite valid token.* +**A:** Verify scope includes the specific operation (`policy:activate` vs `policy:run`). Check tenant header matches token tenant. Inspect Authority logs for denial reason (`policy_scope_denied_total` metric). + +**Q:** *`stella policy run` exits with code `30`.* +**A:** Network/transport error. Check connectivity to Policy Engine endpoint, TLS configuration, and CLI proxy settings. + +**Q:** *Explain drawer shows no VEX data.* +**A:** Either no VEX statement matched or the tenant lacks `findings:read` scope. If VEX should exist, confirm Excititor ingestion and policy joiners (see Observability dashboards). + +--- + +## Compliance Checklist + +- [ ] FAQ linked from Console help menu and CLI `stella policy help`. +- [ ] Entries reviewed quarterly by Policy & Support Guilds. +- [ ] Answers cross-reference lifecycle, runs, observability, and governance docs. +- [ ] Incident/Escalation contact details kept current in Support playbooks. +- [ ] FAQ translated for supported locales (if applicable). + +--- + +*Last updated: 2025-10-26 (Sprint 20).* + diff --git a/docs/ingestion/aggregation-only-contract.md b/docs/ingestion/aggregation-only-contract.md new file mode 100644 index 00000000..4d9c9afd --- /dev/null +++ b/docs/ingestion/aggregation-only-contract.md @@ -0,0 +1,176 @@ +# Aggregation-Only Contract Reference + +> The Aggregation-Only Contract (AOC) is the governing rule set that keeps StellaOps ingestion services deterministic, policy-neutral, and auditable. It applies to Concelier, Excititor, and any future collectors that write raw advisory or VEX documents. + +## 1. Purpose and Scope + +- Defines the canonical behaviour for `advisory_raw` and `vex_raw` collections and the linkset hints they may emit. +- Applies to every ingestion runtime (`StellaOps.Concelier.*`, `StellaOps.Excititor.*`), the Authority scopes that guard them, and the DevOps/QA surfaces that verify compliance. +- Complements the high-level architecture in [Concelier](../ARCHITECTURE_CONCELIER.md) and Authority enforcement documented in [Authority Architecture](../ARCHITECTURE_AUTHORITY.md). +- Paired guidance: see the guard-rail checkpoints in [AOC Guardrails](../aoc/aoc-guardrails.md) and CLI usage that will land in `/docs/cli/` as part of Sprint 19 follow-up. + +## 2. Philosophy and Goals + +- Preserve upstream truth: ingestion only captures immutable raw facts plus provenance, never derived severity or policy decisions. +- Defer interpretation: Policy Engine and downstream overlays remain the sole writers of materialised findings, severity, consensus, or risk scores. +- Make every write explainable: provenance, signatures, and content hashes are required so operators can prove where each fact originated. +- Keep outputs reproducible: identical inputs must yield identical documents, hashes, and linksets across replays and air-gapped installs. + +## 3. Contract Invariants + +| # | Invariant | What it forbids or requires | Enforcement surfaces | +|---|-----------|-----------------------------|----------------------| +| 1 | No derived severity at ingest | Reject top-level keys such as `severity`, `cvss`, `effective_status`, `consensus_provider`, `risk_score`. Raw upstream CVSS remains inside `content.raw`. | Mongo schema validator, `AOCWriteGuard`, Roslyn analyzer, `stella aoc verify`. | +| 2 | No merges or opinionated dedupe | Each upstream document persists on its own; ingestion never collapses multiple vendors into one document. | Repository interceptors, unit/fixture suites. | +| 3 | Provenance is mandatory | `source.*`, `upstream.*`, and `signature` metadata must be present; missing provenance triggers `ERR_AOC_004`. | Schema validator, guard, CLI verifier. | +| 4 | Idempotent upserts | Writes keyed by `(vendor, upstream_id, content_hash)` either no-op or insert a new revision with `supersedes`. Duplicate hashes map to the same document. | Repository guard, storage unique index, CI smoke tests. | +| 5 | Append-only revisions | Updates create a new document with `supersedes` pointer; no in-place mutation of content. | Mongo schema (`supersedes` format), guard, data migration scripts. | +| 6 | Linkset only | Ingestion may compute link hints (`purls`, `cpes`, IDs) to accelerate joins, but must not transform or infer severity or policy. | Linkset builders reviewed via fixtures and analyzers. | +| 7 | Policy-only effective findings | Only Policy Engine identities can write `effective_finding_*`; ingestion callers receive `ERR_AOC_006` if they attempt it. | Authority scopes, Policy Engine guard. | +| 8 | Schema safety | Unknown top-level keys reject with `ERR_AOC_007`; timestamps use ISO 8601 UTC strings; tenant is required. | Mongo validator, JSON schema tests. | +| 9 | Clock discipline | Collectors stamp `fetched_at` and `received_at` monotonically per batch to support reproducibility windows. | Collector contracts, QA fixtures. | + +## 4. Raw Schemas + +### 4.1 `advisory_raw` + +| Field | Type | Notes | +|-------|------|-------| +| `_id` | string | `advisory_raw:{source}:{upstream_id}:{revision}`; deterministic and tenant-scoped. | +| `tenant` | string | Required; injected by Authority middleware and asserted by schema validator. | +| `source.vendor` | string | Provider identifier (e.g., `redhat`, `osv`, `ghsa`). | +| `source.stream` | string | Connector stream name (`csaf`, `osv`, etc.). | +| `source.api` | string | Absolute URI of upstream document; stored for traceability. | +| `source.collector_version` | string | Semantic version of the collector. | +| `upstream.upstream_id` | string | Vendor- or ecosystem-provided identifier (CVE, GHSA, vendor ID). | +| `upstream.document_version` | string | Upstream issued timestamp or revision string. | +| `upstream.fetched_at` / `received_at` | string | ISO 8601 UTC timestamps recorded by the collector. | +| `upstream.content_hash` | string | `sha256:` digest of the raw payload used for idempotency. | +| `upstream.signature` | object | Required structure storing `present`, `format`, `key_id`, `sig`; even unsigned payloads set `present: false`. | +| `content.format` | string | Source format (`CSAF`, `OSV`, etc.). | +| `content.spec_version` | string | Upstream spec version when known. | +| `content.raw` | object | Full upstream payload, untouched except for transport normalisation. | +| `identifiers` | object | Normalised identifiers (`cve`, `ghsa`, `aliases`, etc.) derived losslessly from raw content. | +| `linkset` | object | Join hints (see section 4.3). | +| `supersedes` | string or null | Points to previous revision of same upstream doc when content hash changes. | + +### 4.2 `vex_raw` + +| Field | Type | Notes | +|-------|------|-------| +| `_id` | string | `vex_raw:{source}:{upstream_id}:{revision}`. | +| `tenant` | string | Required; matches advisory collection requirements. | +| `source.*` | object | Same shape and requirements as `advisory_raw`. | +| `upstream.*` | object | Includes `document_version`, timestamps, `content_hash`, and `signature`. | +| `content.format` | string | Typically `CycloneDX-VEX` or `CSAF-VEX`. | +| `content.raw` | object | Entire upstream VEX payload. | +| `identifiers.statements` | array | Normalised statement summaries (IDs, PURLs, status, justification) to accelerate policy joins. | +| `linkset` | object | CVEs, GHSA IDs, and PURLs referenced in the document. | +| `supersedes` | string or null | Same convention as advisory documents. | + +### 4.3 Linkset Fields + +- `purls`: fully qualified Package URLs extracted from raw ranges or product nodes. +- `cpes`: Common Platform Enumerations when upstream docs provide them. +- `aliases`: Any alternate advisory identifiers present in the payload. +- `references`: Array of `{ type, url }` pairs pointing back to vendor advisories, patches, or exploits. +- `reconciled_from`: Provenance of linkset entries (JSON Pointer or field origin) to make automated checks auditable. + +### 4.4 `advisory_observations` + +`advisory_observations` is an immutable projection of the validated raw document used by Link‑Not‑Merge overlays. Fields mirror the JSON contract surfaced by `StellaOps.Concelier.Models.Observations.AdvisoryObservation`. + +| Field | Type | Notes | +|-------|------|-------| +| `_id` | string | Deterministic observation id — `{tenant}:{source.vendor}:{upstreamId}:{revision}`. | +| `tenant` | string | Lower-case tenant identifier. | +| `source.vendor` / `source.stream` | string | Connector identity (e.g., `vendor/redhat`, `ecosystem/osv`). | +| `source.api` | string | Absolute URI the connector fetched from. | +| `source.collectorVersion` | string | Optional semantic version of the connector build. | +| `upstream.upstream_id` | string | Advisory identifier as issued by the provider (CVE, vendor ID, etc.). | +| `upstream.document_version` | string | Upstream revision/version string. | +| `upstream.fetchedAt` / `upstream.receivedAt` | datetime | UTC timestamps recorded by the connector. | +| `upstream.contentHash` | string | `sha256:` digest used for idempotency. | +| `upstream.signature` | object | `{present, format?, keyId?, signature?}` describing upstream signature material. | +| `content.format` / `content.specVersion` | string | Raw payload format metadata (CSAF, OSV, JSON, etc.). | +| `content.raw` | object | Full upstream document stored losslessly (Relaxed Extended JSON). | +| `content.metadata` | object | Optional connector-specific metadata (batch ids, hints). | +| `linkset.aliases` | array | Normalized aliases (lower-case, sorted). | +| `linkset.purls` | array | Normalized PURLs extracted from the document. | +| `linkset.cpes` | array | Normalized CPE URIs. | +| `linkset.references` | array | `{ type, url }` pairs (type lower-case). | +| `createdAt` | datetime | Timestamp when Concelier persisted the observation. | +| `attributes` | object | Optional provenance attributes keyed by connector. | + +## 5. Error Model + +| Code | Description | HTTP status | Surfaces | +|------|-------------|-------------|----------| +| `ERR_AOC_001` | Forbidden field detected (severity, cvss, effective data). | 400 | Ingestion APIs, CLI verifier, CI guard. | +| `ERR_AOC_002` | Merge attempt detected (multiple upstream sources fused into one document). | 400 | Ingestion APIs, CLI verifier. | +| `ERR_AOC_003` | Idempotency violation (duplicate without supersedes pointer). | 409 | Repository guard, Mongo unique index, CLI verifier. | +| `ERR_AOC_004` | Missing provenance metadata (`source`, `upstream`, `signature`). | 422 | Schema validator, ingestion endpoints. | +| `ERR_AOC_005` | Signature or checksum mismatch. | 422 | Collector validation, CLI verifier. | +| `ERR_AOC_006` | Attempt to persist derived findings from ingestion context. | 403 | Policy engine guard, Authority scopes. | +| `ERR_AOC_007` | Unknown top-level fields (schema violation). | 400 | Mongo validator, CLI verifier. | + +Consumers should map these codes to CLI exit codes and structured log events so automation can fail fast and produce actionable guidance. + +## 6. API and Tooling Interfaces + +- **Concelier ingestion** (`StellaOps.Concelier.WebService`) + - `POST /ingest/advisory`: accepts upstream payload metadata; server-side guard constructs and persists raw document. + - `GET /advisories/raw/{id}` and filterable list endpoints expose raw documents for debugging and offline analysis. + - `POST /aoc/verify`: runs guard checks over recent documents and returns summary totals plus first violations. +- **Excititor ingestion** (`StellaOps.Excititor.WebService`) mirrors the same surface for VEX documents. +- **CLI workflows** (`stella aoc verify`, `stella sources ingest --dry-run`) surface pre-flight verification; documentation will live in `/docs/cli/` alongside Sprint 19 CLI updates. +- **Authority scopes**: new `advisory:write`, `advisory:verify`, `vex:write`, and `vex:verify` scopes enforce least privilege; see [Authority Architecture](../ARCHITECTURE_AUTHORITY.md) for scope grammar. + +## 7. Idempotency and Supersedes Rules + +1. Compute `content_hash` before any transformation; use it with `(source.vendor, upstream.upstream_id)` to detect duplicates. +2. If a document with the same hash already exists, skip the write and log a no-op. +3. When a new hash arrives for an existing upstream document, insert a new record and set `supersedes` to the previous `_id`. +4. Keep supersedes chains acyclic; collectors must resolve conflicts by rewinding before they insert. +5. Expose idempotency counters via metrics (`ingestion_write_total{result=ok|noop}`) to catch regressions early. + +## 8. Migration Playbook + +1. Freeze ingestion writes except for raw pass-through paths while deploying schema validators. +2. Snapshot existing collections to `_backup_*` for rollback safety. +3. Strip forbidden fields from historical documents into a temporary `advisory_view_legacy` used only during transition. +4. Enable Mongo JSON schema validators for `advisory_raw` and `vex_raw`. +5. Run collectors in `--dry-run` to confirm only allowed keys appear; fix violations before lifting the freeze. +6. Point Policy Engine to consume exclusively from raw collections and compute derived outputs downstream. +7. Delete legacy normalisation paths from ingestion code and enable runtime guards plus CI linting. +8. Roll forward CLI, Console, and dashboards so operators can monitor AOC status end-to-end. + +## 9. Observability and Diagnostics + +- **Metrics**: `ingestion_write_total{result=ok|reject}`, `aoc_violation_total{code}`, `ingestion_signature_verified_total{result}`, `ingestion_latency_seconds`, `advisory_revision_count`. +- **Traces**: spans `ingest.fetch`, `ingest.transform`, `ingest.write`, and `aoc.guard` with correlation IDs shared across workers. +- **Logs**: structured entries must include `tenant`, `source.vendor`, `upstream.upstream_id`, `content_hash`, and `violation_code` when applicable. +- **Dashboards**: DevOps should add panels for violation counts, signature failures, supersedes growth, and CLI verifier outcomes for each tenant. + +## 10. Security and Tenancy Checklist + +- Enforce Authority scopes (`advisory:write`, `vex:write`, `advisory:verify`, `vex:verify`) and require tenant claims on every request. +- Maintain pinned trust stores for signature verification; capture verification result in metrics and logs. +- Ensure collectors never log secrets or raw authentication headers; redact tokens before persistence. +- Validate that Policy Engine remains the only identity with permission to write `effective_finding_*` documents. +- Verify offline bundles include the raw collections, guard configuration, and verifier binaries so air-gapped installs can audit parity. +- Document operator steps for recovering from violations, including rollback to superseded revisions and re-running policy evaluation. + +## 11. Compliance Checklist + +- [ ] Deterministic guard enabled in Concelier and Excititor repositories. +- [ ] Mongo validators deployed for `advisory_raw` and `vex_raw`. +- [ ] Authority scopes and tenant enforcement verified via integration tests. +- [ ] CLI and CI pipelines run `stella aoc verify` against seeded snapshots. +- [ ] Observability feeds (metrics, logs, traces) wired into dashboards with alerts. +- [ ] Offline kit instructions updated to bundle validators and verifier tooling. +- [ ] Security review recorded covering ingestion, tenancy, and rollback procedures. + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/observability/observability.md b/docs/observability/observability.md new file mode 100644 index 00000000..083e14ef --- /dev/null +++ b/docs/observability/observability.md @@ -0,0 +1,141 @@ +# AOC Observability Guide + +> **Audience:** Observability Guild, Concelier/Excititor SREs, platform operators. +> **Scope:** Metrics, traces, logs, dashboards, and runbooks introduced as part of the Aggregation-Only Contract (AOC) rollout (Sprint 19). + +This guide captures the canonical signals emitted by Concelier and Excititor once AOC guards are active. It explains how to consume the metrics in dashboards, correlate traces/logs for incident triage, and operate in offline environments. Pair this guide with the [AOC reference](../ingestion/aggregation-only-contract.md) and [architecture overview](../architecture/overview.md). + +--- + +## 1 · Metrics + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `ingestion_write_total` | Counter | `source`, `tenant`, `result` (`ok`, `reject`, `noop`) | Counts write attempts to `advisory_raw`/`vex_raw`. Rejects correspond to guard failures. | +| `ingestion_latency_seconds` | Histogram | `source`, `tenant`, `phase` (`fetch`, `transform`, `write`) | Measures end-to-end runtime for ingestion stages. Use `quantile=0.95` for alerting. | +| `aoc_violation_total` | Counter | `source`, `tenant`, `code` (`ERR_AOC_00x`) | Total guard violations bucketed by error code. Drives dashboard pills and alert thresholds. | +| `ingestion_signature_verified_total` | Counter | `source`, `tenant`, `result` (`ok`, `fail`, `skipped`) | Tracks signature/checksum verification outcomes. | +| `advisory_revision_count` | Gauge | `source`, `tenant` | Supersedes depth for raw documents; spikes indicate noisy upstream feeds. | +| `verify_runs_total` | Counter | `tenant`, `initiator` (`ui`, `cli`, `api`, `scheduled`) | How many `stella aoc verify` or `/aoc/verify` runs executed. | +| `verify_duration_seconds` | Histogram | `tenant`, `initiator` | Runtime of verification jobs; use P95 to detect regressions. | + +### 1.1 Alerts + +- **Violation spike:** Alert when `increase(aoc_violation_total[15m]) > 0` for critical sources. Page SRE if `code="ERR_AOC_005"` (signature failure) or `ERR_AOC_001` persists > 30 min. +- **Stale ingestion:** Alert when `max_over_time(ingestion_latency_seconds_sum / ingestion_latency_seconds_count)[30m]` exceeds 30 s or if `ingestion_write_total` has no growth for > 60 min. +- **Signature drop:** Warn when `rate(ingestion_signature_verified_total{result="fail"}[1h]) > 0`. + +--- + +## 2 · Traces + +### 2.1 Span taxonomy + +| Span name | Parent | Key attributes | +|-----------|--------|----------------| +| `ingest.fetch` | job root span | `source`, `tenant`, `uri`, `contentHash` | +| `ingest.transform` | `ingest.fetch` | `documentType` (`csaf`, `osv`, `vex`), `payloadBytes` | +| `ingest.write` | `ingest.transform` | `collection` (`advisory_raw`, `vex_raw`), `result` (`ok`, `reject`) | +| `aoc.guard` | `ingest.write` | `code` (on violation), `violationCount`, `supersedes` | +| `verify.run` | verification job root | `tenant`, `window.from`, `window.to`, `sources`, `violations` | + +### 2.2 Trace usage + +- Correlate UI dashboard entries with traces via `traceId` surfaced in violation drawers (`docs/ui/console.md`). +- Use `aoc.guard` spans to inspect guard payload snapshots. Sensitive fields are redacted automatically; raw JSON lives in secure logs only. +- For scheduled verification, filter traces by `initiator="scheduled"` to compare runtimes pre/post change. + +--- + +## 3 · Logs + +Structured logs include the following keys (JSON): + +| Key | Description | +|-----|-------------| +| `traceId` | Matches OpenTelemetry trace/span IDs for cross-system correlation. | +| `tenant` | Tenant identifier enforced by Authority middleware. | +| `source.vendor` | Logical source (e.g., `redhat`, `ubuntu`, `osv`, `ghsa`). | +| `upstream.upstreamId` | Vendor-provided ID (CVE, GHSA, etc.). | +| `contentHash` | `sha256:` digest of the raw document. | +| `violation.code` | Present when guard rejects `ERR_AOC_00x`. | +| `verification.window` | Present on `/aoc/verify` job logs. | + +Logs are shipped to the central Loki/Elasticsearch cluster. Use the template query: + +```logql +{app="concelier-web"} | json | violation_code != "" +``` + +to spot active AOC violations. + +--- + +## 4 · Dashboards + +Primary Grafana dashboard: **“AOC Ingestion Health”** (`dashboards/aoc-ingestion.json`). Panels include: + +1. **Sources overview:** table fed by `ingestion_write_total` and `ingestion_latency_seconds` (mirrors Console tiles). +2. **Violation trend:** stacked bar chart of `aoc_violation_total` per code. +3. **Signature success rate:** timeseries derived from `ingestion_signature_verified_total`. +4. **Supersedes depth:** gauge showing `advisory_revision_count` P95. +5. **Verification runs:** histogram and latency boxplot using `verify_runs_total` / `verify_duration_seconds`. + +Secondary dashboards: + +- **AOC Alerts (Ops view):** summarises active alerts, last verify run, and links to incident runbook. +- **Offline Mode Dashboard:** fed from Offline Kit imports; highlights snapshot age and queued verification jobs. + +Update `docs/assets/dashboards/` with screenshots when Grafana capture pipeline produces the latest renders. + +--- + +## 5 · Operational workflows + +1. **During ingestion incident:** + - Check Console dashboard for offending sources. + - Pivot to logs using document `contentHash`. + - Re-run `stella sources ingest --dry-run` with problematic payloads to validate fixes. + - After remediation, run `stella aoc verify --since 24h` and confirm exit code `0`. +2. **Scheduled verification:** + - Configure cron job to run `stella aoc verify --format json --export ...`. + - Ship JSON to `aoc-verify` bucket and ingest into metrics using custom exporter. + - Alert on missing exports (no file uploaded within 26 h). +3. **Offline kit validation:** + - Use Offline Dashboard to ensure snapshots contain latest metrics. + - Run verification reports locally and attach to bundle before distribution. + +--- + +## 6 · Offline considerations + +- Metrics exporters bundled with Offline Kit write to local Prometheus snapshots; sync them with central Grafana once connectivity is restored. +- CLI verification reports should be hashed (`sha256sum`) and archived for audit trails. +- Dashboards include offline data sources (`prometheus-offline`) switchable via dropdown. + +--- + +## 7 · References + +- [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) +- [Architecture overview](../architecture/overview.md) +- [Console AOC dashboard](../ui/console.md) +- [CLI AOC commands](../cli/cli-reference.md) +- [Concelier architecture](../ARCHITECTURE_CONCELIER.md) +- [Excititor architecture](../ARCHITECTURE_EXCITITOR.md) + +--- + +## 8 · Compliance checklist + +- [ ] Metrics documented with label sets and alert guidance. +- [ ] Tracing span taxonomy aligned with Concelier/Excititor implementation. +- [ ] Log schema matches structured logging contracts (traceId, tenant, source, contentHash). +- [ ] Grafana dashboard references verified and screenshots scheduled. +- [ ] Offline/air-gap workflow captured. +- [ ] Cross-links to AOC reference, console, and CLI docs included. +- [ ] Observability Guild sign-off scheduled (OWNER: @obs-guild, due 2025-10-28). + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/observability/policy.md b/docs/observability/policy.md new file mode 100644 index 00000000..7e5091e8 --- /dev/null +++ b/docs/observability/policy.md @@ -0,0 +1,166 @@ +# Policy Engine Observability + +> **Audience:** Observability Guild, SRE/Platform operators, Policy Guild. +> **Scope:** Metrics, logs, traces, dashboards, alerting, sampling, and incident workflows for the Policy Engine service (Sprint 20). +> **Prerequisites:** Policy Engine v2 deployed with OpenTelemetry exporters enabled (`observability:enabled=true` in config). + +--- + +## 1 · Instrumentation Overview + +- **Telemetry stack:** OpenTelemetry SDK (metrics + traces), Serilog structured logging, OTLP exporters → Collector → Prometheus/Loki/Tempo. +- **Namespace conventions:** `policy.*` for metrics/traces/log categories; labels use `tenant`, `policy`, `mode`, `runId`. +- **Sampling:** Default 10 % trace sampling, 1 % rule-hit log sampling; incident mode overrides to 100 % (see §6). +- **Correlation IDs:** Every API request gets `traceId` + `requestId`. CLI/UI display IDs to streamline support. + +--- + +## 2 · Metrics + +### 2.1 Run Pipeline + +| Metric | Type | Labels | Notes | +|--------|------|--------|-------| +| `policy_run_seconds` | Histogram | `tenant`, `policy`, `mode` (`full`, `incremental`, `simulate`) | P95 target ≤ 5 min incremental, ≤ 30 min full. | +| `policy_run_queue_depth` | Gauge | `tenant` | Number of pending jobs per tenant (updated each enqueue/dequeue). | +| `policy_run_failures_total` | Counter | `tenant`, `policy`, `reason` (`err_pol_*`, `network`, `cancelled`) | Aligns with error codes. | +| `policy_run_retries_total` | Counter | `tenant`, `policy` | Helps identify noisy sources. | +| `policy_run_inputs_pending_bytes` | Gauge | `tenant` | Size of buffered change batches awaiting run. | + +### 2.2 Evaluator Insights + +| Metric | Type | Labels | Notes | +|--------|------|--------|-------| +| `policy_rules_fired_total` | Counter | `tenant`, `policy`, `rule` | Increment per rule match (sampled). | +| `policy_vex_overrides_total` | Counter | `tenant`, `policy`, `vendor`, `justification` | Tracks VEX precedence decisions. | +| `policy_suppressions_total` | Counter | `tenant`, `policy`, `action` (`ignore`, `warn`, `quiet`) | Audits suppression usage. | +| `policy_selection_batch_duration_seconds` | Histogram | `tenant`, `policy` | Measures joiner performance. | +| `policy_materialization_conflicts_total` | Counter | `tenant`, `policy` | Non-zero indicates optimistic concurrency retries. | + +### 2.3 API Surface + +| Metric | Type | Labels | Notes | +|--------|------|--------|-------| +| `policy_api_requests_total` | Counter | `endpoint`, `method`, `status` | Exposed via Minimal API instrumentation. | +| `policy_api_latency_seconds` | Histogram | `endpoint`, `method` | Budget ≤ 250 ms for GETs, ≤ 1 s for POSTs. | +| `policy_api_rate_limited_total` | Counter | `endpoint` | Tied to throttles (`429`). | + +### 2.4 Queue & Change Streams + +| Metric | Type | Labels | Notes | +|--------|------|--------|-------| +| `policy_queue_leases_active` | Gauge | `tenant` | Number of leased jobs. | +| `policy_queue_lease_expirations_total` | Counter | `tenant` | Alerts when workers fail to ack. | +| `policy_delta_backlog_age_seconds` | Gauge | `tenant`, `source` (`concelier`, `excititor`, `sbom`) | Age of oldest unprocessed change event. | + +--- + +## 3 · Logs + +- **Format:** JSON (`Serilog`). Core fields: `timestamp`, `level`, `message`, `policyId`, `policyVersion`, `tenant`, `runId`, `rule`, `traceId`, `env.sealed`, `error.code`. +- **Log categories:** + - `policy.run` (queue lifecycle, run begin/end, stats) + - `policy.evaluate` (batch execution summaries; rule-hit sampling) + - `policy.materialize` (Mongo operations, conflicts, retries) + - `policy.simulate` (diff results, CLI invocation metadata) + - `policy.lifecycle` (submit/review/approve events) +- **Sampling:** Rule-hit logs sample 1 % by default; toggled to 100 % in incident mode or when `--trace` flag used in CLI. +- **PII:** No user secrets recorded; user identities referenced as `user:` or `group:` only. + +--- + +## 4 · Traces + +- Spans emit via OpenTelemetry instrumentation. +- **Primary spans:** + - `policy.api` – wraps HTTP request, records `endpoint`, `status`, `scope`. + - `policy.select` – change stream ingestion and batch assembly (attributes: `candidateCount`, `cursor`). + - `policy.evaluate` – evaluation batch (attributes: `batchSize`, `ruleHits`, `severityChanges`). + - `policy.materialize` – Mongo writes (attributes: `writes`, `historyWrites`, `retryCount`). + - `policy.simulate` – simulation diff generation (attributes: `sbomCount`, `diffAdded`, `diffRemoved`). +- Trace context propagated to CLI via response headers `traceparent`; UI surfaces in run detail view. +- Incident mode forces span sampling to 100 % and extends retention via Collector config override. + +--- + +## 5 · Dashboards + +### 5.1 Policy Runs Overview + +Widgets: +- Run duration histogram (per mode/tenant). +- Queue depth + backlog age line charts. +- Failure rate stacked by error code. +- Incremental backlog heatmap (policy × age). +- Active vs scheduled runs table. + +### 5.2 Rule Impact & VEX + +- Top N rules by firings (bar chart). +- VEX overrides by vendor/justification (stacked chart). +- Suppression usage (pie + table with justifications). +- Quieted findings trend (line). + +### 5.3 Simulation & Approval Health + +- Simulation diff histogram (added vs removed). +- Pending approvals by age (table with SLA colour coding). +- Compliance checklist status (lint, determinism CI, simulation evidence). + +> Placeholders for Grafana panels should be replaced with actual screenshots once dashboards land (`../assets/policy-observability/*.png`). + +--- + +## 6 · Alerting + +| Alert | Condition | Suggested Action | +|-------|-----------|------------------| +| **PolicyRunSlaBreach** | `policy_run_seconds{mode="incremental"}` P95 > 300 s for 3 windows | Check queue depth, upstream services, scale worker pool. | +| **PolicyQueueStuck** | `policy_delta_backlog_age_seconds` > 600 | Investigate change stream connectivity. | +| **DeterminismMismatch** | Run status `failed` with `ERR_POL_004` OR CI replay diff | Switch to incident sampling, gather replay bundle, notify Policy Guild. | +| **SimulationDrift** | CLI/CI simulation exit `20` (blocking diff) over threshold | Review policy changes before approval. | +| **VexOverrideSpike** | `policy_vex_overrides_total` > configured baseline (per vendor) | Verify upstream VEX feed; ensure justification codes expected. | +| **SuppressionSurge** | `policy_suppressions_total` increase > 3σ vs baseline | Audit new suppress rules; check approvals. | + +Alerts integrate with Notifier channels (`policy.alerts`) and Ops on-call rotations. + +--- + +## 7 · Incident Mode & Forensics + +- Toggle via `POST /api/policy/incidents/activate` (requires `policy:operate` scope). +- Effects: + - Trace sampling → 100 %. + - Rule-hit log sampling → 100 %. + - Retention window extended to 30 days for incident duration. + - `policy.incident.activated` event emitted (Console + Notifier banners). +- Post-incident tasks: + - `stella policy run replay` for affected runs; attach bundles to incident record. + - Restore sampling defaults with `.../deactivate`. + - Update incident checklist in `/docs/policy/lifecycle.md` (section 8) with findings. + +--- + +## 8 · Integration Points + +- **Authority:** Exposes metric `policy_scope_denied_total` for failed authorisation; correlate with `policy_api_requests_total`. +- **Concelier/Excititor:** Shared trace IDs propagate via gRPC metadata to help debug upstream latency. +- **Scheduler:** Future integration will push run queues into shared scheduler dashboards (planned in SCHED-MODELS-20-002). +- **Offline Kit:** CLI exports logs + metrics snapshots (`stella offline bundle metrics`) for air-gapped audits. + +--- + +## 9 · Compliance Checklist + +- [ ] **Metrics registered:** All metrics listed above exported and documented in Grafana dashboards. +- [ ] **Alert policies configured:** Ops or Observability Guild created alerts matching table in §6. +- [ ] **Sampling overrides tested:** Incident mode toggles verified in staging; retention roll-back rehearsed. +- [ ] **Trace propagation validated:** CLI/UI display trace IDs and allow copy for support. +- [ ] **Log scrubbing enforced:** Unit tests guarantee no secrets/PII in logs; sampling respects configuration. +- [ ] **Offline capture rehearsed:** Metrics/log snapshot commands executed in sealed environment. +- [ ] **Docs cross-links:** Links to architecture, runs, lifecycle, CLI, API docs verified. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* + diff --git a/docs/ops/concelier-authority-audit-runbook.md b/docs/ops/concelier-authority-audit-runbook.md index ccaeacc2..c09b46b0 100644 --- a/docs/ops/concelier-authority-audit-runbook.md +++ b/docs/ops/concelier-authority-audit-runbook.md @@ -1,152 +1,159 @@ -# Concelier Authority Audit Runbook - +# Concelier Authority Audit Runbook + _Last updated: 2025-10-22_ - -This runbook helps operators verify and monitor the StellaOps Concelier ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity. - -## 1. Prerequisites - + +This runbook helps operators verify and monitor the StellaOps Concelier ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity. + +## 1. Prerequisites + - Authority integration is enabled in `concelier.yaml` (or via `CONCELIER_AUTHORITY__*` environment variables) with a valid `clientId`, secret, audience, and required scopes. - OTLP metrics/log exporters are configured (`concelier.telemetry.*`) or container stdout is shipped to your SIEM. - Operators have access to the Concelier job trigger endpoints via CLI or REST for smoke tests. - The rollout table in `docs/10_CONCELIER_CLI_QUICKSTART.md` has been reviewed so stakeholders align on the staged → enforced toggle timeline. - -### Configuration snippet - -```yaml -concelier: - authority: - enabled: true - allowAnonymousFallback: false # keep true only during initial rollout - issuer: "https://authority.internal" - audiences: - - "api://concelier" - requiredScopes: - - "concelier.jobs.trigger" - bypassNetworks: - - "127.0.0.1/32" - - "::1/128" - clientId: "concelier-jobs" - clientSecretFile: "/run/secrets/concelier_authority_client" - tokenClockSkewSeconds: 60 - resilience: - enableRetries: true - retryDelays: - - "00:00:01" - - "00:00:02" - - "00:00:05" - allowOfflineCacheFallback: true - offlineCacheTolerance: "00:10:00" -``` - -> Store secrets outside source control. Concelier reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service. - -### Resilience tuning - -- **Connected sites:** keep the default 1 s / 2 s / 5 s retry ladder so Concelier retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts. -- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (15–30 minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Concelier will fail fast but keep deterministic logs. -- Concelier resolves these knobs through `IOptionsMonitor`. Edits to `concelier.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled. - -## 2. Key Signals - -### 2.1 Audit log channel - -Concelier emits structured audit entries via the `Concelier.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active. - -``` -Concelier authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=concelier-cli scopes=concelier.jobs.trigger bypass=False remote=10.1.4.7 -``` - -| Field | Sample value | Meaning | -|--------------|-------------------------|------------------------------------------------------------------------------------------| -| `route` | `/jobs/definitions` | Endpoint that processed the request. | -| `status` | `200` / `401` / `409` | Final HTTP status code returned to the caller. | -| `subject` | `ops@example.com` | User or service principal subject (falls back to `(anonymous)` when unauthenticated). | -| `clientId` | `concelier-cli` | OAuth client ID provided by Authority ( `(none)` if the token lacked the claim). | -| `scopes` | `concelier.jobs.trigger` | Normalised scope list extracted from token claims; `(none)` if the token carried none. | -| `bypass` | `True` / `False` | Indicates whether the request succeeded because its source IP matched a bypass CIDR. | -| `remote` | `10.1.4.7` | Remote IP recorded from the connection / forwarded header test hooks. | - -Use your logging backend (e.g., Loki) to index the logger name and filter for suspicious combinations: - -- `status=401 AND bypass=True` – bypass network accepted an unauthenticated call (should be temporary during rollout). -- `status=202 AND scopes="(none)"` – a token without scopes triggered a job; tighten client configuration. -- Spike in `clientId="(none)"` – indicates upstream Authority is not issuing `client_id` claims or the CLI is outdated. - -### 2.2 Metrics - -Concelier publishes counters under the OTEL meter `StellaOps.Concelier.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`. - -| Metric name | Description | PromQL example | -|-------------------------------|----------------------------------------------------|----------------| -| `web.jobs.triggered` | Accepted job trigger requests. | `sum by (job_kind) (rate(web_jobs_triggered_total[5m]))` | -| `web.jobs.trigger.conflict` | Rejected triggers (already running, disabled…). | `sum(rate(web_jobs_trigger_conflict_total[5m]))` | -| `web.jobs.trigger.failed` | Server-side job failures. | `sum(rate(web_jobs_trigger_failed_total[5m]))` | - -> Prometheus/OTEL collectors typically surface counters with `_total` suffix. Adjust queries to match your pipeline’s generated metric names. - -Correlate audit logs with the following global meter exported via `Concelier.SourceDiagnostics`: - -- `concelier.source.http.requests_total{concelier_source="jobs-run"}` – ensures REST/manual triggers route through Authority. -- If Grafana dashboards are deployed, extend the “Concelier Jobs” board with the above counters plus a table of recent audit log entries. - -## 3. Alerting Guidance - -1. **Unauthorized bypass attempt** - - Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", status="401", bypass="True"}[5m])) > 0` - - Action: verify `bypassNetworks` list; confirm expected maintenance windows; rotate credentials if suspicious. - -2. **Missing scopes** - - Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0` - - Action: audit Authority client registration; ensure `requiredScopes` includes `concelier.jobs.trigger`. - -3. **Trigger failure surge** - - Query: `sum(rate(web_jobs_trigger_failed_total[10m])) > 0` with severity `warning` if sustained for 10 minutes. - - Action: inspect correlated audit entries and `Concelier.Telemetry` traces for job execution errors. - -4. **Conflict spike** - - Query: `sum(rate(web_jobs_trigger_conflict_total[10m])) > 5` (tune threshold). - - Action: downstream scheduling may be firing repetitive triggers; ensure precedence is configured properly. - -5. **Authority offline** - - Watch `Concelier.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback. - -## 4. Rollout & Verification Procedure - + +### Configuration snippet + +```yaml +concelier: + authority: + enabled: true + allowAnonymousFallback: false # keep true only during initial rollout + issuer: "https://authority.internal" + audiences: + - "api://concelier" + requiredScopes: + - "concelier.jobs.trigger" + - "advisory:read" + - "advisory:ingest" + requiredTenants: + - "tenant-default" + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + clientId: "concelier-jobs" + clientSecretFile: "/run/secrets/concelier_authority_client" + tokenClockSkewSeconds: 60 + resilience: + enableRetries: true + retryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + allowOfflineCacheFallback: true + offlineCacheTolerance: "00:10:00" +``` + +> Store secrets outside source control. Concelier reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service. + +### Resilience tuning + +- **Connected sites:** keep the default 1 s / 2 s / 5 s retry ladder so Concelier retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts. +- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (15–30 minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Concelier will fail fast but keep deterministic logs. +- Concelier resolves these knobs through `IOptionsMonitor`. Edits to `concelier.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled. + +## 2. Key Signals + +### 2.1 Audit log channel + +Concelier emits structured audit entries via the `Concelier.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active. + +``` +Concelier authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=concelier-cli scopes=concelier.jobs.trigger advisory:ingest bypass=False remote=10.1.4.7 +``` + +| Field | Sample value | Meaning | +|--------------|-------------------------|------------------------------------------------------------------------------------------| +| `route` | `/jobs/definitions` | Endpoint that processed the request. | +| `status` | `200` / `401` / `409` | Final HTTP status code returned to the caller. | +| `subject` | `ops@example.com` | User or service principal subject (falls back to `(anonymous)` when unauthenticated). | +| `clientId` | `concelier-cli` | OAuth client ID provided by Authority (`(none)` if the token lacked the claim). | +| `scopes` | `concelier.jobs.trigger advisory:ingest advisory:read` | Normalised scope list extracted from token claims; `(none)` if the token carried none. | +| `tenant` | `tenant-default` | Tenant claim extracted from the Authority token (`(none)` when the token lacked it). | +| `bypass` | `True` / `False` | Indicates whether the request succeeded because its source IP matched a bypass CIDR. | +| `remote` | `10.1.4.7` | Remote IP recorded from the connection / forwarded header test hooks. | + +Use your logging backend (e.g., Loki) to index the logger name and filter for suspicious combinations: + +- `status=401 AND bypass=True` – bypass network accepted an unauthenticated call (should be temporary during rollout). +- `status=202 AND scopes="(none)"` – a token without scopes triggered a job; tighten client configuration. +- `status=202 AND NOT contains(scopes,"advisory:ingest")` – ingestion attempted without the new AOC scopes; confirm the Authority client registration matches the sample above. +- `tenant!=(tenant-default)` – indicates a cross-tenant token was accepted. Ensure Concelier `requiredTenants` is aligned with Authority client registration. +- Spike in `clientId="(none)"` – indicates upstream Authority is not issuing `client_id` claims or the CLI is outdated. + +### 2.2 Metrics + +Concelier publishes counters under the OTEL meter `StellaOps.Concelier.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`. + +| Metric name | Description | PromQL example | +|-------------------------------|----------------------------------------------------|----------------| +| `web.jobs.triggered` | Accepted job trigger requests. | `sum by (job_kind) (rate(web_jobs_triggered_total[5m]))` | +| `web.jobs.trigger.conflict` | Rejected triggers (already running, disabled…). | `sum(rate(web_jobs_trigger_conflict_total[5m]))` | +| `web.jobs.trigger.failed` | Server-side job failures. | `sum(rate(web_jobs_trigger_failed_total[5m]))` | + +> Prometheus/OTEL collectors typically surface counters with `_total` suffix. Adjust queries to match your pipeline’s generated metric names. + +Correlate audit logs with the following global meter exported via `Concelier.SourceDiagnostics`: + +- `concelier.source.http.requests_total{concelier_source="jobs-run"}` – ensures REST/manual triggers route through Authority. +- If Grafana dashboards are deployed, extend the “Concelier Jobs” board with the above counters plus a table of recent audit log entries. + +## 3. Alerting Guidance + +1. **Unauthorized bypass attempt** + - Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", status="401", bypass="True"}[5m])) > 0` + - Action: verify `bypassNetworks` list; confirm expected maintenance windows; rotate credentials if suspicious. + +2. **Missing scopes** + - Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0` + - Action: audit Authority client registration; ensure `requiredScopes` includes `concelier.jobs.trigger`, `advisory:ingest`, and `advisory:read`. + +3. **Trigger failure surge** + - Query: `sum(rate(web_jobs_trigger_failed_total[10m])) > 0` with severity `warning` if sustained for 10 minutes. + - Action: inspect correlated audit entries and `Concelier.Telemetry` traces for job execution errors. + +4. **Conflict spike** + - Query: `sum(rate(web_jobs_trigger_conflict_total[10m])) > 5` (tune threshold). + - Action: downstream scheduling may be firing repetitive triggers; ensure precedence is configured properly. + +5. **Authority offline** + - Watch `Concelier.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback. + +## 4. Rollout & Verification Procedure + 1. **Pre-checks** - Align with the rollout phases documented in `docs/10_CONCELIER_CLI_QUICKSTART.md` (validation → rehearsal → enforced) and record the target dates in your change request. - Confirm `allowAnonymousFallback` is `false` in production; keep `true` only during staged validation. - Validate Authority issuer metadata is reachable from Concelier (`curl https://authority.internal/.well-known/openid-configuration` from the host). - -2. **Smoke test with valid token** - - Obtain a token via CLI: `stella auth login --scope concelier.jobs.trigger`. - - Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://concelier.internal/jobs/definitions`. - - Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=concelier.jobs.trigger`. - -3. **Negative test without token** - - Call the same endpoint without a token. Expect HTTP 401, `bypass=False`. - - If the request succeeds, double-check `bypassNetworks` and ensure fallback is disabled. - -4. **Bypass check (if applicable)** - - From an allowed maintenance IP, call `/jobs/definitions` without a token. Confirm the audit log shows `bypass=True`. Review business justification and expiry date for such entries. - -5. **Metrics validation** - - Ensure `web.jobs.triggered` counter increments during accepted runs. - - Exporters should show corresponding spans (`concelier.job.trigger`) if tracing is enabled. - -## 5. Troubleshooting - -| Symptom | Probable cause | Remediation | -|---------|----------------|-------------| -| Audit log shows `clientId=(none)` for all requests | Authority not issuing `client_id` claim or CLI outdated | Update StellaOps Authority configuration (`StellaOpsAuthorityOptions.Token.Claims.ClientId`), or upgrade the CLI token acquisition flow. | -| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Concelier. | -| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`concelier.jobs.trigger`) and ensure the token audience matches `audiences` config. | -| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `concelier.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Concelier.WebService.Jobs` meter. | -| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Concelier job logs, re-run with tracing enabled, validate Authority latency. | - -## 6. References - -- `docs/21_INSTALL_GUIDE.md` – Authority configuration quick start. -- `docs/17_SECURITY_HARDENING_GUIDE.md` – Security guardrails and enforcement deadlines. -- `docs/ops/authority-monitoring.md` – Authority-side monitoring and alerting playbook. -- `StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs` – source of audit log fields. + +2. **Smoke test with valid token** + - Obtain a token via CLI: `stella auth login --scope "concelier.jobs.trigger advisory:ingest" --scope advisory:read`. + - Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://concelier.internal/jobs/definitions`. + - Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=concelier.jobs.trigger advisory:ingest advisory:read`, and `tenant=tenant-default`. + +3. **Negative test without token** + - Call the same endpoint without a token. Expect HTTP 401, `bypass=False`. + - If the request succeeds, double-check `bypassNetworks` and ensure fallback is disabled. + +4. **Bypass check (if applicable)** + - From an allowed maintenance IP, call `/jobs/definitions` without a token. Confirm the audit log shows `bypass=True`. Review business justification and expiry date for such entries. + +5. **Metrics validation** + - Ensure `web.jobs.triggered` counter increments during accepted runs. + - Exporters should show corresponding spans (`concelier.job.trigger`) if tracing is enabled. + +## 5. Troubleshooting + +| Symptom | Probable cause | Remediation | +|---------|----------------|-------------| +| Audit log shows `clientId=(none)` for all requests | Authority not issuing `client_id` claim or CLI outdated | Update StellaOps Authority configuration (`StellaOpsAuthorityOptions.Token.Claims.ClientId`), or upgrade the CLI token acquisition flow. | +| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Concelier. | +| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`concelier.jobs.trigger`) and ensure the token audience matches `audiences` config. | +| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `concelier.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Concelier.WebService.Jobs` meter. | +| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Concelier job logs, re-run with tracing enabled, validate Authority latency. | + +## 6. References + +- `docs/21_INSTALL_GUIDE.md` – Authority configuration quick start. +- `docs/17_SECURITY_HARDENING_GUIDE.md` – Security guardrails and enforcement deadlines. +- `docs/ops/authority-monitoring.md` – Authority-side monitoring and alerting playbook. +- `StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs` – source of audit log fields. diff --git a/docs/ops/deployment-upgrade-runbook.md b/docs/ops/deployment-upgrade-runbook.md new file mode 100644 index 00000000..7fd585e2 --- /dev/null +++ b/docs/ops/deployment-upgrade-runbook.md @@ -0,0 +1,151 @@ +# Stella Ops Deployment Upgrade & Rollback Runbook + +_Last updated: 2025-10-26 (Sprint 14 – DEVOPS-OPS-14-003)._ + +This runbook describes how to promote a new release across the supported deployment profiles (Helm and Docker Compose), how to roll back safely, and how to keep channels (`edge`, `stable`, `airgap`) aligned. All steps assume you are working from a clean checkout of the release branch/tag. + +--- + +## 1. Channel overview + +| Channel | Release manifest | Helm values | Compose profile | +|---------|------------------|-------------|-----------------| +| `edge` | `deploy/releases/2025.10-edge.yaml` | `deploy/helm/stellaops/values-dev.yaml` | `deploy/compose/docker-compose.dev.yaml` | +| `stable` | `deploy/releases/2025.09-stable.yaml` | `deploy/helm/stellaops/values-stage.yaml`, `deploy/helm/stellaops/values-prod.yaml` | `deploy/compose/docker-compose.stage.yaml`, `deploy/compose/docker-compose.prod.yaml` | +| `airgap` | `deploy/releases/2025.09-airgap.yaml` | `deploy/helm/stellaops/values-airgap.yaml` | `deploy/compose/docker-compose.airgap.yaml` | + +Infrastructure components (MongoDB, MinIO, RustFS) are pinned in the release manifests and inherited by the deployment profiles. Supporting dependencies such as `nats` remain on upstream LTS tags; review `deploy/compose/*.yaml` for the authoritative set. + +--- + +## 2. Pre-flight checklist + +1. **Refresh release manifest** + Pull the latest manifest for the channel you are promoting (`deploy/releases/-.yaml`). + +2. **Align deployment bundles with the manifest** + Run the alignment checker for every profile that should pick up the release. Pass `--ignore-repo nats` to skip auxiliary services. + ```bash + ./deploy/tools/check-channel-alignment.py \ + --release deploy/releases/2025.10-edge.yaml \ + --target deploy/helm/stellaops/values-dev.yaml \ + --target deploy/compose/docker-compose.dev.yaml \ + --ignore-repo nats + ``` + Repeat for other channels (`stable`, `airgap`), substituting the manifest and target files. + +3. **Lint and template profiles** + ```bash + ./deploy/tools/validate-profiles.sh + ``` + +4. **Smoke the Offline Kit debug store (edge/stable only)** + When the release pipeline has generated `out/release/debug/.build-id/**`, mirror the assets into the Offline Kit staging tree: + ```bash + ./ops/offline-kit/mirror_debug_store.py \ + --release-dir out/release \ + --offline-kit-dir out/offline-kit + ``` + Archive the resulting `out/offline-kit/metadata/debug-store.json` alongside the kit bundle. + +5. **Review compatibility matrix** + Confirm MongoDB, MinIO, and RustFS versions in the release manifest match platform SLOs. The default targets are `mongo@sha256:c258…`, `minio@sha256:14ce…`, `rustfs:2025.10.0-edge`. + +6. **Create a rollback bookmark** + Record the current Helm revision (`helm history stellaops -n stellaops`) and compose tag (`git describe --tags`) before applying changes. + +--- + +## 3. Helm upgrade procedure (staging → production) + +1. Switch to the deployment branch and ensure secrets/config maps are current. +2. Apply the upgrade in the staging cluster: + ```bash + helm upgrade stellaops deploy/helm/stellaops \ + -f deploy/helm/stellaops/values-stage.yaml \ + --namespace stellaops \ + --atomic \ + --timeout 15m + ``` +3. Run smoke tests (`scripts/smoke-tests.sh` or environment-specific checks). +4. Promote to production using the prod values file and the same command. +5. Record the new revision number and Git SHA in the change log. + +### Rollback (Helm) + +1. Identify the previous revision: `helm history stellaops -n stellaops`. +2. Execute: + ```bash + helm rollback stellaops \ + --namespace stellaops \ + --wait \ + --timeout 10m + ``` +3. Verify `kubectl get pods` returns healthy workloads; rerun smoke tests. +4. Update the incident/operations log with root cause and rollback details. + +--- + +## 4. Docker Compose upgrade procedure + +1. Update environment files (`deploy/compose/env/*.env.example`) with any new settings and sync secrets to hosts. +2. Pull the tagged repository state corresponding to the release (e.g. `git checkout 2025.09.2` for stable). +3. Apply the upgrade: + ```bash + docker compose \ + --env-file deploy/compose/env/prod.env \ + -f deploy/compose/docker-compose.prod.yaml \ + pull + + docker compose \ + --env-file deploy/compose/env/prod.env \ + -f deploy/compose/docker-compose.prod.yaml \ + up -d + ``` +4. Tail logs for critical services (`docker compose logs -f authority concelier`). +5. Update monitoring dashboards/alerts to confirm normal operation. + +### Rollback (Compose) + +1. Check out the previous release tag (e.g. `git checkout 2025.09.1`). +2. Re-run `docker compose pull` and `docker compose up -d` with that profile. Docker will restore the prior digests. +3. If reverting to a known-good snapshot is required, restore volume backups (see `docs/ops/authority-backup-restore.md` and associated service guides). +4. Log the rollback in the operations journal. + +--- + +## 5. Channel promotion workflow + +1. Author or update the channel manifest under `deploy/releases/`. +2. Mirror the new digests into Helm/Compose values and run the alignment script for each profile. +3. Commit the changes with a message that references the release version and channel (e.g. `deploy: promote 2025.10.0-edge`). +4. Publish release notes and update `deploy/releases/README.md` (if applicable). +5. Tag the repository when promoting stable or airgap builds. + +--- + +## 6. Upgrade rehearsal & rollback drill log + +Maintain rehearsal notes in `docs/ops/launch-cutover.md` or the relevant sprint planning document. After each drill capture: + +- Release version tested +- Date/time +- Participants +- Issues encountered & fixes +- Rollback duration (if executed) + +Attach the log to the sprint retro or operational wiki. + +| Date (UTC) | Channel | Outcome | Notes | +|------------|---------|---------|-------| +| 2025-10-26 | Documentation dry-run | Planned | Runbook refreshed; next live drill scheduled for 2025-11 edge → stable promotion. + +--- + +## 7. References + +- `deploy/README.md` – structure and validation workflow for deployment bundles. +- `docs/13_RELEASE_ENGINEERING_PLAYBOOK.md` – release automation and signing pipeline. +- `docs/ARCHITECTURE_DEVOPS.md` – high-level DevOps architecture, SLOs, and compliance requirements. +- `ops/offline-kit/mirror_debug_store.py` – debug-store mirroring helper. +- `deploy/tools/check-channel-alignment.py` – release vs deployment digest alignment checker. diff --git a/docs/ops/launch-cutover.md b/docs/ops/launch-cutover.md new file mode 100644 index 00000000..fc807975 --- /dev/null +++ b/docs/ops/launch-cutover.md @@ -0,0 +1,128 @@ +# Launch Cutover Runbook - Stella Ops + +_Document owner: DevOps Guild (2025-10-26)_ +_Scope:_ Full-platform launch from staging to production for release `2025.09.2`. + +## 1. Roles and Communication + +| Role | Primary | Backup | Contact | +| --- | --- | --- | --- | +| Cutover lead | DevOps Guild (on-call engineer) | Platform Ops lead | `#launch-bridge` (Mattermost) | +| Authority stack | Authority Core guild rep | Security guild rep | `#authority` | +| Scanner / Queue | Scanner WebService guild rep | Runtime guild rep | `#scanner` | +| Storage | Mongo/MinIO operators | Backup DB admin | Pager escalation | +| Observability | Telemetry guild rep | SRE on-call | `#telemetry` | +| Approvals | Product owner + CTO | DevOps lead | Approval recorded in change ticket | + +Set up a bridge call 30 minutes before start and keep `#launch-bridge` updated every 10 minutes. + +## 2. Timeline Overview (UTC) + +| Time | Activity | Owner | +| --- | --- | --- | +| T-24h | Change ticket approved, prod secrets verified, offline kit build status checked (`DEVOPS-OFFLINE-18-005`). | DevOps lead | +| T-12h | Run `deploy/tools/validate-profiles.sh`; capture logs in ticket. | DevOps engineer | +| T-6h | Freeze non-launch deployments; notify guild leads. | Product owner | +| T-2h | Execute rehearsal in staging (Section 3) using `values-stage.yaml` to verify scripts. | DevOps + module reps | +| T-30m | Final go/no-go with guild leads; confirm monitoring dashboards green. | Cutover lead | +| T0 | Execute production cutover steps (Section 4). | Cutover team | +| T+45m | Smoke tests complete (Section 5); announce success or trigger rollback. | Cutover lead | +| T+4h | Post-cutover metrics review, notify stakeholders, close ticket. | DevOps + product owner | + +## 3. Rehearsal (Staging) Checklist + +1. `docker network create stellaops_frontdoor || true` (if not present on staging jump host). +2. Run `deploy/tools/validate-profiles.sh` and archive output. +3. Apply staging secrets (`kubectl apply -f secrets/stage/*.yaml` or `helm secrets upgrade`) ensuring `stellaops-stage` credentials align with `values-stage.yaml`. +4. Perform `helm upgrade stellaops deploy/helm/stellaops -f deploy/helm/stellaops/values-stage.yaml` in staging cluster. +5. Verify health endpoints: `curl https://authority.stage.../healthz`, `curl https://scanner.stage.../healthz`. +6. Execute smoke CLI: `stellaops-cli scan submit --profile staging --sbom samples/sbom/demo.json` and confirm report status in UI. +7. Document total wall time and any deviations in the rehearsal log. + +Rehearsal must complete without manual interventions before proceeding to production. + +## 4. Production Cutover Steps + +### 4.1 Pre-flight +- Confirm production secrets in the appropriate secret store (`stellaops-prod-core`, `stellaops-prod-mongo`, `stellaops-prod-minio`, `stellaops-prod-notify`) contain the keys referenced in `values-prod.yaml`. +- Ensure the external reverse proxy network exists: `docker network create stellaops_frontdoor || true` on each compose host. +- Back up current configuration and data: + - Mongo snapshot: `mongodump --uri "$MONGO_BACKUP_URI" --out /backups/launch-$(date -Iseconds)`. + - MinIO policy export: `mc mirror --overwrite minio/stellaops minio-backup/stellaops-$(date +%Y%m%d%H%M)`. + +### 4.2 Apply Updates (Compose) +1. On each compose node, pull updated images for release `2025.09.2`: + ```bash + docker compose --env-file prod.env -f deploy/compose/docker-compose.prod.yaml pull + ``` +2. Deploy changes: + ```bash + docker compose --env-file prod.env -f deploy/compose/docker-compose.prod.yaml up -d + ``` +3. Confirm containers healthy via `docker compose ps` and `docker logs --tail 50`. + +### 4.3 Apply Updates (Helm/Kubernetes) +If using Kubernetes, perform: +```bash +helm upgrade stellaops deploy/helm/stellaops -f deploy/helm/stellaops/values-prod.yaml --atomic --timeout 15m +``` +Monitor rollout with `kubectl get pods -n stellaops --watch` and `kubectl rollout status deployment/`. + +### 4.4 Configuration Validation +- Verify Authority issuer metadata: `curl https://authority.prod.../.well-known/openid-configuration`. +- Validate Signer DSSE endpoint: `stellaops-cli signer verify --base-url https://signer.prod... --bundle samples/dsse/demo.json`. +- Check Scanner queue connectivity: `docker exec stellaops-scanner-web dotnet StellaOps.Scanner.WebService.dll health queue` (returns success). +- Ensure Notify (legacy) still accessible while Notifier migration pending. + +## 5. Smoke Tests + +| Test | Command / Action | Expected Result | +| --- | --- | --- | +| API health | `curl https://scanner.prod.../healthz` | HTTP 200 with `status":"Healthy"` | +| Scan submit | `stellaops-cli scan submit --profile prod --sbom samples/sbom/demo.json` | Scan completes < 5 minutes; report accessible with signed DSSE | +| Runtime event ingest | Post sample event from Zastava observer fixture | `/runtime/events` responds 202 Accepted; record visible in Mongo `runtime_events` | +| Signing | `stellaops-cli signer sign --bundle demo.json` | Returns DSSE with matching SHA256 and signer metadata | +| Attestor verify | `stellaops-cli attestor verify --uuid ` | Verification result `ok=true` | +| Web UI | Manual login, verify dashboards render and latency within budget | UI loads under 2 seconds; policy views consistent | + +Log results in the change ticket with timestamps and screenshots where applicable. + +## 6. Rollback Procedure + +1. Assess failure scope; if systemic, initiate rollback immediately while preserving logs/artifacts. +2. For Compose: + ```bash + docker compose --env-file prod.env -f deploy/compose/docker-compose.prod.yaml down + docker compose --env-file stage.env -f deploy/compose/docker-compose.stage.yaml up -d + ``` +3. For Helm: + ```bash + helm rollback stellaops --namespace stellaops + ``` +4. Restore Mongo snapshot if data inconsistency detected: `mongorestore --uri "$MONGO_BACKUP_URI" --drop /backups/launch-`. +5. Restore MinIO mirror if required: `mc mirror minio-backup/stellaops- minio/stellaops`. +6. Notify stakeholders of rollback and capture root cause notes in incident ticket. + +## 7. Post-cutover Actions + +- Keep heightened monitoring for 4 hours post cutover; track latency, error rates, and queue depth. +- Confirm audit trails: Authority tokens issued, Scanner events recorded, Attestor submissions stored. +- Update `docs/ops/launch-readiness.md` if any new gaps or follow-ups discovered. +- Schedule retrospective within 48 hours; include DevOps, module guilds, and product owner. + +## 8. Approval Matrix + +| Step | Required Approvers | Record Location | +| --- | --- | --- | +| Production deployment plan | CTO + DevOps lead | Change ticket comment | +| Cutover start (T0) | DevOps lead + module reps | `#launch-bridge` summary | +| Post-smoke success | DevOps lead + product owner | Change ticket closure | +| Rollback (if invoked) | DevOps lead + CTO | Incident ticket | + +Retain all approvals and logs for audit. Update this runbook after each execution to record actual timings and lessons learned. + +## 9. Rehearsal Log + +| Date (UTC) | What We Exercised | Outcome | Follow-up | +| --- | --- | --- | --- | +| 2025-10-26 | Dry-run of compose/Helm validation via `deploy/tools/validate-profiles.sh` (dev/stage/prod/airgap/mirror). Network creation simulated (`docker network create stellaops_frontdoor` planned) and stage CLI submission reviewed. | Validation script succeeded; all profiles templated cleanly. Stage deployment apply deferred because no staging cluster is accessible from the current environment. | Schedule full stage rehearsal once staging cluster credentials are available; reuse this log section to capture timings. | diff --git a/docs/ops/launch-readiness.md b/docs/ops/launch-readiness.md new file mode 100644 index 00000000..61ca387f --- /dev/null +++ b/docs/ops/launch-readiness.md @@ -0,0 +1,49 @@ +# Launch Readiness Record - Stella Ops + +_Updated: 2025-10-26 (UTC)_ + +This document captures production launch sign-offs, deployment readiness checkpoints, and any open risks that must be tracked before GA cutover. + +## 1. Sign-off Summary + +| Module / Service | Guild / Point of Contact | Evidence (Task or Runbook) | Status | Timestamp (UTC) | Notes | +| --- | --- | --- | --- | --- | --- | +| Authority (Issuer) | Authority Core Guild | `AUTH-AOC-19-001` - scope issuance & configuration complete (DONE 2025-10-26) | READY | 2025-10-26T14:05Z | Tenant scope propagation follow-up (`AUTH-AOC-19-002`) tracked in gaps section. | +| Signer | Signer Guild | `SIGNER-API-11-101` / `SIGNER-REF-11-102` / `SIGNER-QUOTA-11-103` (DONE 2025-10-21) | READY | 2025-10-26T14:07Z | DSSE signing, referrer verification, and quota enforcement validated in CI. | +| Attestor | Attestor Guild | `ATTESTOR-API-11-201` / `ATTESTOR-VERIFY-11-202` / `ATTESTOR-OBS-11-203` (DONE 2025-10-19) | READY | 2025-10-26T14:10Z | Rekor submission/verification pipeline green; telemetry pack published. | +| Scanner Web + Worker | Scanner WebService Guild | `SCANNER-WEB-09-10x`, `SCANNER-RUNTIME-12-30x` (DONE 2025-10-18 -> 2025-10-24) | READY* | 2025-10-26T14:20Z | Orchestrator envelope work (`SCANNER-EVENTS-16-301/302`) still open; see gaps. | +| Concelier Core & Connectors | Concelier Core / Ops Guild | Ops runbook sign-off in `docs/ops/concelier-conflict-resolution.md` (2025-10-16) | READY | 2025-10-26T14:25Z | Conflict resolution & connector coverage accepted; Mongo schema hardening pending (see gaps). | +| Excititor API | Excititor Core Guild | Wave 0 connector ingest sign-offs (EXECPLAN.Section Wave 0) | READY | 2025-10-26T14:28Z | VEX linkset publishing complete for launch datasets. | +| Notify Web (legacy) | Notify Guild | Existing stack carried forward; Notifier program tracked separately (Sprint 38-40) | PENDING | 2025-10-26T14:32Z | Legacy notify web remains operational; migration to Notifier blocked on `SCANNER-EVENTS-16-301`. | +| Web UI | UI Guild | Stable build `registry.stella-ops.org/.../web-ui@sha256:10d9248...` deployed in stage and smoke-tested | READY | 2025-10-26T14:35Z | Policy editor GA items (Sprint 20) outside launch scope. | +| DevOps / Release | DevOps Guild | `deploy/tools/validate-profiles.sh` run (2025-10-26) covering dev/stage/prod/airgap/mirror | READY | 2025-10-26T15:02Z | Compose/Helm lint + docker compose config validated; see Section 2 for details. | +| Offline Kit | Offline Kit Guild | `DEVOPS-OFFLINE-18-004` (Go analyzer) and `DEVOPS-OFFLINE-18-005` (Python analyzer) complete; debug-store mirror pending (`DEVOPS-OFFLINE-17-004`). | PENDING | 2025-10-26T15:05Z | Awaiting release debug artefacts to finalise `DEVOPS-OFFLINE-17-004`; tracked in Section 3. | + +_\* READY with caveat - remaining work noted in Section 3._ + +## 2. Deployment Readiness Checklist + +- **Production profiles committed:** `deploy/compose/docker-compose.prod.yaml` and `deploy/helm/stellaops/values-prod.yaml` added with front-door network hand-off and secret references for Mongo/MinIO/core services. +- **Secrets placeholders documented:** `deploy/compose/env/prod.env.example` enumerates required credentials (`MONGO_INITDB_ROOT_PASSWORD`, `MINIO_ROOT_PASSWORD`, Redis/NATS endpoints, `FRONTDOOR_NETWORK`). Helm values reference Kubernetes secrets (`stellaops-prod-core`, `stellaops-prod-mongo`, `stellaops-prod-minio`, `stellaops-prod-notify`). +- **Static validation executed:** `deploy/tools/validate-profiles.sh` run on 2025-10-26 (docker compose config + helm lint/template) with all profiles passing. +- **Ingress model defined:** Production compose profile introduces external `frontdoor` network; README updated with creation instructions and scope of externally reachable services. +- **Observability hooks:** Authority/Signer/Attestor telemetry packs verified; scanner runtime build-id metrics landed (`SCANNER-RUNTIME-17-401`). Grafana dashboards referenced in component runbooks. +- **Rollback assets:** Stage Compose profile remains aligned (`docker-compose.stage.yaml`), enabling rehearsals before prod cutover; release manifests (`deploy/releases/2025.09-stable.yaml`) map digests for reproducible rollback. +- **Rehearsal status:** 2025-10-26 validation dry-run executed (`deploy/tools/validate-profiles.sh` across dev/stage/prod/airgap/mirror). Full stage Helm rollout pending access to the managed staging cluster; target to complete once credentials are provisioned. + +## 3. Outstanding Gaps & Follow-ups + +| Item | Owner | Tracking Ref | Target / Next Step | Impact | +| --- | --- | --- | --- | --- | +| Tenant scope propagation and audit coverage | Authority Core Guild | `AUTH-AOC-19-002` (DOING 2025-10-26) | Land enforcement + audit fixtures by Sprint 19 freeze | Medium - required for multi-tenant GA but does not block initial cutover if tenants scoped manually. | +| Orchestrator event envelopes + Notifier handshake | Scanner WebService Guild | `SCANNER-EVENTS-16-301` (BLOCKED), `SCANNER-EVENTS-16-302` (DOING) | Coordinate with Gateway/Notifier owners on preview package replacement or binding redirects; rerun `dotnet test` once patch lands and refresh schema docs. Share envelope samples in `docs/events/` after tests pass. | High — gating Notifier migration; legacy notify path remains functional meanwhile. | +| Offline Kit Python analyzer bundle | Offline Kit Guild + Scanner Guild | `DEVOPS-OFFLINE-18-005` (DONE 2025-10-26) | Monitor for follow-up manifest updates and rerun smoke script when analyzers change. | Medium - ensures language analyzer coverage stays current for offline installs. | +| Offline Kit debug store mirror | Offline Kit Guild + DevOps Guild | `DEVOPS-OFFLINE-17-004` (BLOCKED 2025-10-26) | Release pipeline must publish `out/release/debug` artefacts; once available, run `mirror_debug_store.py` and commit `metadata/debug-store.json`. | Low - symbol lookup remains accessible from staging assets but required before next Offline Kit tag. | +| Mongo schema validators for advisory ingestion | Concelier Storage Guild | `CONCELIER-STORE-AOC-19-001` (TODO) | Finalize JSON schema + migration toggles; coordinate with Ops for rollout window | Low - current validation handled in app layer; schema guard adds defense-in-depth. | +| Authority plugin telemetry alignment | Security Guild | `SEC2.PLG`, `SEC3.PLG`, `SEC5.PLG` (BLOCKED pending AUTH DPoP/MTLS tasks) | Resume once upstream auth surfacing stabilises | Low - plugin remains optional; launch uses default Authority configuration. | + +## 4. Approvals & Distribution + +- Record shared in `#launch-readiness` (Mattermost) 2025-10-26 15:15 UTC with DevOps + Guild leads for acknowledgement. +- Updates to this document require dual sign-off from DevOps Guild (owner) and impacted module guild lead; retain change log via Git history. +- Cutover rehearsal and rollback drills are tracked separately in `docs/ops/launch-cutover.md` (see associated Task `DEVOPS-LAUNCH-18-001`). *** End Patch diff --git a/docs/ops/nuget-preview-bootstrap.md b/docs/ops/nuget-preview-bootstrap.md index 8227d545..46593dbd 100644 --- a/docs/ops/nuget-preview-bootstrap.md +++ b/docs/ops/nuget-preview-bootstrap.md @@ -1,6 +1,6 @@ # NuGet Preview Bootstrap (Offline-Friendly) -The StellaOps build relies on .NET 10 preview packages (Microsoft.Extensions.*, JwtBearer 10.0 RC). +The StellaOps build relies on .NET 10 RC2 packages (Microsoft.Extensions.*, JwtBearer 10.0 RC). `NuGet.config` now wires three sources: 1. `local` → `./local-nuget` (preferred, air-gapped mirror) @@ -32,6 +32,21 @@ DOTNET_NOLOGO=1 dotnet restore src/StellaOps.Excititor.Connectors.Abstractions/S The `packageSourceMapping` section keeps `Microsoft.Extensions.*`, `Microsoft.AspNetCore.*`, and `Microsoft.Data.Sqlite` bound to `local`/`dotnet-public`, so `dotnet restore` never has to reach out to nuget.org when mirrors are populated. +Before committing changes (or when wiring up a new environment) run: + +```bash +python3 ops/devops/validate_restore_sources.py +``` + +The validator asserts: + +- `NuGet.config` lists `local` → `dotnet-public` → `nuget.org` in that order. +- `Directory.Build.props` pins `RestoreSources` so every project prioritises the local mirror. +- No stray `NuGet.config` files shadow the repo root configuration. + +CI executes the validator in both the `build-test-deploy` and `release` workflows, +so regressions trip before any restore/build begins. + If you run fully air-gapped, remember to clear the cache between SDK upgrades: ```bash diff --git a/docs/ops/registry-token-service.md b/docs/ops/registry-token-service.md new file mode 100644 index 00000000..5e17419b --- /dev/null +++ b/docs/ops/registry-token-service.md @@ -0,0 +1,66 @@ +# Registry Token Service Operations + +_Component_: `src/StellaOps.Registry.TokenService` + +The registry token service issues short-lived Docker registry bearer tokens after +validating an Authority OpTok (DPoP/mTLS sender constraint) and the customer’s +plan entitlements. It is fronted by the Docker registry’s `Bearer realm` flow. + +## Configuration + +Configuration lives in `etc/registry-token.yaml` and can be overridden through +environment variables prefixed with `REGISTRY_TOKEN_`. Key sections: + +| Section | Purpose | +| ------- | ------- | +| `authority` | Authority issuer/metadata URL, audience list, and scopes required to request tokens (default `registry.token.issue`). | +| `signing` | JWT issuer, signing key (PEM or PFX), optional key ID, and token lifetime (default five minutes). The repository ships **`etc/registry-signing-sample.pem`** for local testing only – replace it with a private key generated and stored per-environment before going live. | +| `registry` | Registry realm URL and optional allow-list of `service` values accepted from the registry challenge. | +| `plans` | Plan catalogue mapping plan name → repository patterns and allowed actions. Wildcards (`*`) are supported per path segment. | +| `defaultPlan` | Applied when the caller’s token omits `stellaops:plan`. | +| `revokedLicenses` | Blocks issuance when the caller presents a matching `stellaops:license` claim. | + +Plan entries must cover every private repository namespace. Actions default to +`pull` if omitted. + +## Request flow + +1. Docker/OCI client contacts the registry and receives a `401` with + `WWW-Authenticate: Bearer realm=...,service=...,scope=repository:...`. +2. Client acquires an OpTok from Authority (DPoP/mTLS bound) with the + `registry.token.issue` scope. +3. Client calls `GET /token?service=&scope=repository::` + against the token service, presenting the OpTok and matching DPoP proof. +4. The service validates the token, plan, and requested scopes, then issues a + JWT containing an `access` claim conforming to the Docker registry spec. + +All denial paths return RFC 6750-style problem responses (HTTP 400 for malformed +scopes, 403 for plan or revocation failures). + +## Monitoring + +The service emits OpenTelemetry metrics via `registry_token_issued_total` and +`registry_token_rejected_total`. Suggested Prometheus alerts: + +| Metric | Condition | Action | +|--------|-----------|--------| +| `registry_token_rejected_total` | `increase(...) > 0` over 5 minutes | Investigate plan misconfiguration or licence revocation. | +| `registry_token_issued_total` | Sudden drop compared to baseline | Confirm registry is still challenging with the expected realm/service. | + +Enable the built-in `/healthz` endpoint for liveness checks. Authentication and +DPoP failures surface via the service logs (Serilog console output). + +## Sample deployment + +```bash +dotnet run --project src/StellaOps.Registry.TokenService \ + --urls "http://0.0.0.0:8085" + +curl -H "Authorization: Bearer " \ + -H "DPoP: $(dpop-proof ...)" \ + "http://localhost:8085/token?service=registry.localhost&scope=repository:stella-ops/public/base:pull" +``` + +Replace `` and `DPoP` with tokens issued by Authority. The response +contains `token`, `expires_in`, and `issued_at` fields suitable for Docker/OCI +clients. diff --git a/docs/ops/telemetry-collector.md b/docs/ops/telemetry-collector.md new file mode 100644 index 00000000..588115f6 --- /dev/null +++ b/docs/ops/telemetry-collector.md @@ -0,0 +1,113 @@ +# Telemetry Collector Deployment Guide + +> **Scope:** DevOps Guild, Observability Guild, and operators enabling the StellaOps telemetry pipeline (DEVOPS-OBS-50-001 / DEVOPS-OBS-50-003). + +This guide describes how to deploy the default OpenTelemetry Collector packaged with Stella Ops, validate its ingest endpoints, and prepare an offline-ready bundle for air-gapped environments. + +--- + +## 1. Overview + +The collector terminates OTLP traffic from Stella Ops services and exports metrics, traces, and logs. + +| Endpoint | Purpose | TLS | Authentication | +| -------- | ------- | --- | -------------- | +| `:4317` | OTLP gRPC ingest | mTLS | Client certificate issued by collector CA | +| `:4318` | OTLP HTTP ingest | mTLS | Client certificate issued by collector CA | +| `:9464` | Prometheus scrape | mTLS | Same client certificate | +| `:13133` | Health check | mTLS | Same client certificate | +| `:1777` | pprof diagnostics | mTLS | Same client certificate | + +The default configuration lives at `deploy/telemetry/otel-collector-config.yaml` and mirrors the Helm values in the `stellaops` chart. + +--- + +## 2. Local validation (Compose) + +```bash +# 1. Generate dev certificates (CA + collector + client) +./ops/devops/telemetry/generate_dev_tls.sh + +# 2. Start the collector overlay +cd deploy/compose +docker compose -f docker-compose.telemetry.yaml up -d + +# 3. Start the storage overlay (Prometheus, Tempo, Loki) +docker compose -f docker-compose.telemetry-storage.yaml up -d + +# 4. Run the smoke test (OTLP HTTP) +python ../../ops/devops/telemetry/smoke_otel_collector.py --host localhost +``` + +The smoke test posts sample traces, metrics, and logs and verifies that the collector increments the `otelcol_receiver_accepted_*` counters exposed via the Prometheus exporter. The storage overlay gives you a local Prometheus/Tempo/Loki stack to confirm end-to-end wiring. The same client certificate can be used by local services to weave traces together. See [`Telemetry Storage Deployment`](telemetry-storage.md) for the storage configuration guidelines used in staging/production. + +--- + +## 3. Kubernetes deployment + +Enable the collector in Helm by setting the following values (example shown for the dev profile): + +```yaml +telemetry: + collector: + enabled: true + defaultTenant: + tls: + secretName: stellaops-otel-tls- +``` + +Provide a Kubernetes secret named `stellaops-otel-tls-` (for staging: `stellaops-otel-tls-stage`) with the keys `tls.crt`, `tls.key`, and `ca.crt`. The secret must contain the collector certificate, private key, and issuing CA respectively. Example: + +```bash +kubectl create secret generic stellaops-otel-tls-stage \ + --from-file=tls.crt=collector.crt \ + --from-file=tls.key=collector.key \ + --from-file=ca.crt=ca.crt +``` + +Helm renders the collector deployment, service, and config map automatically: + +```bash +helm upgrade --install stellaops deploy/helm/stellaops -f deploy/helm/stellaops/values-dev.yaml +``` + +Update client workloads to trust `ca.crt` and present client certificates that chain back to the same CA. + +--- + +## 4. Offline packaging (DEVOPS-OBS-50-003) + +Use the packaging helper to produce a tarball that can be mirrored inside the Offline Kit or air-gapped sites: + +```bash +python ops/devops/telemetry/package_offline_bundle.py --output out/telemetry/telemetry-bundle.tar.gz +``` + +The script gathers: + +- `deploy/telemetry/README.md` +- Collector configuration (`deploy/telemetry/otel-collector-config.yaml` and Helm copy) +- Helm template/values for the collector +- Compose overlay (`deploy/compose/docker-compose.telemetry.yaml`) + +The tarball ships with a `.sha256` checksum. To attach a Cosign signature, add `--sign` and provide `COSIGN_KEY_REF`/`COSIGN_IDENTITY_TOKEN` env vars (or use the `--cosign-key` flag). + +Distribute the bundle alongside certificates generated by your PKI. For air-gapped installs, regenerate certificates inside the enclave and recreate the `stellaops-otel-tls` secret. + +--- + +## 5. Operational checks + +1. **Health probes** – `kubectl exec` into the collector pod and run `curl -fsSk --cert client.crt --key client.key --cacert ca.crt https://127.0.0.1:13133/healthz`. +2. **Metrics scrape** – confirm Prometheus ingests `otelcol_receiver_accepted_*` counters. +3. **Trace correlation** – ensure services propagate `trace_id` and `tenant.id` attributes; refer to `docs/observability/observability.md` for expected spans. +4. **Certificate rotation** – when rotating the CA, update the secret and restart the collector; roll out new client certificates before enabling `require_client_certificate` if staged. + +--- + +## 6. Related references + +- `deploy/telemetry/README.md` – source configuration and local workflow. +- `ops/devops/telemetry/smoke_otel_collector.py` – OTLP smoke test. +- `docs/observability/observability.md` – metrics/traces/logs taxonomy. +- `docs/13_RELEASE_ENGINEERING_PLAYBOOK.md` – release checklist for telemetry assets. diff --git a/docs/ops/telemetry-storage.md b/docs/ops/telemetry-storage.md new file mode 100644 index 00000000..153d8f36 --- /dev/null +++ b/docs/ops/telemetry-storage.md @@ -0,0 +1,172 @@ +# Telemetry Storage Deployment (DEVOPS-OBS-50-002) + +> **Audience:** DevOps Guild, Observability Guild +> +> **Scope:** Prometheus (metrics), Tempo (traces), Loki (logs) storage backends with tenant isolation, TLS, retention policies, and Authority integration. + +--- + +## 1. Components & Ports + +| Service | Port | Purpose | TLS | +|-----------|------|---------|-----| +| Prometheus | 9090 | Metrics API / alerting | Client auth (mTLS) to scrape collector | +| Tempo | 3200 | Trace ingest + API | mTLS (client cert required) | +| Loki | 3100 | Log ingest + API | mTLS (client cert required) | + +The collector forwards OTLP traffic to Tempo (traces), Prometheus scrapes the collector’s `/metrics` endpoint, and Loki is used for log search. + +--- + +## 2. Local validation (Compose) + +```bash +./ops/devops/telemetry/generate_dev_tls.sh +cd deploy/compose +# Start collector + storage stack +docker compose -f docker-compose.telemetry.yaml up -d +docker compose -f docker-compose.telemetry-storage.yaml up -d +python ../../ops/devops/telemetry/smoke_otel_collector.py --host localhost +``` + +Configuration files live in `deploy/telemetry/storage/`. Adjust the overrides before shipping to staging/production. + +--- + +## 3. Kubernetes blueprint + +Deploy Prometheus, Tempo, and Loki to the `observability` namespace. The Helm values snippet below illustrates the key settings (charts not yet versioned—define them in the observability repo): + +```yaml +prometheus: + server: + extraFlags: + - web.enable-lifecycle + persistentVolume: + enabled: true + size: 200Gi + additionalScrapeConfigsSecret: stellaops-prometheus-scrape + extraSecretMounts: + - name: otel-mtls + secretName: stellaops-otel-tls-stage + mountPath: /etc/telemetry/tls + readOnly: true + - name: otel-token + secretName: stellaops-prometheus-token + mountPath: /etc/telemetry/auth + readOnly: true + +loki: + auth_enabled: true + singleBinary: + replicas: 2 + storage: + type: filesystem + existingSecretForTls: stellaops-otel-tls-stage + runtimeConfig: + configMap: + name: stellaops-loki-tenant-overrides + +tempo: + server: + http_listen_port: 3200 + storage: + trace: + backend: s3 + s3: + endpoint: tempo-minio.observability.svc:9000 + bucket: tempo-traces + multitenancyEnabled: true + extraVolumeMounts: + - name: otel-mtls + mountPath: /etc/telemetry/tls + readOnly: true + - name: tempo-tenant-overrides + mountPath: /etc/telemetry/tenants + readOnly: true +``` + +### Staging bootstrap commands + +```bash +kubectl create namespace observability --dry-run=client -o yaml | kubectl apply -f - + +# TLS material (generated via ops/devops/telemetry/generate_dev_tls.sh or from PKI) +kubectl -n observability create secret generic stellaops-otel-tls-stage \ + --from-file=tls.crt=collector-stage.crt \ + --from-file=tls.key=collector-stage.key \ + --from-file=ca.crt=collector-ca.crt + +# Prometheus bearer token issued by Authority (scope obs:read) +kubectl -n observability create secret generic stellaops-prometheus-token \ + --from-file=token=prometheus-stage.token + +# Tenant overrides +kubectl -n observability create configmap stellaops-loki-tenant-overrides \ + --from-file=overrides.yaml=deploy/telemetry/storage/tenants/loki-overrides.yaml + +kubectl -n observability create configmap tempo-tenant-overrides \ + --from-file=tempo-overrides.yaml=deploy/telemetry/storage/tenants/tempo-overrides.yaml + +# Additional scrape config referencing the collector service +kubectl -n observability create secret generic stellaops-prometheus-scrape \ + --from-file=prometheus-additional.yaml=deploy/telemetry/storage/prometheus.yaml +``` + +Provision the following secrets/configs (names can be overridden via Helm values): + +| Name | Type | Notes | +|------|------|-------| +| `stellaops-otel-tls-stage` | Secret | Shared CA + server cert/key for collector/storage mTLS. +| `stellaops-prometheus-token` | Secret | Bearer token minted by Authority (`obs:read`). +| `stellaops-loki-tenant-overrides` | ConfigMap | Text from `deploy/telemetry/storage/tenants/loki-overrides.yaml`. +| `tempo-tenant-overrides` | ConfigMap | Text from `deploy/telemetry/storage/tenants/tempo-overrides.yaml`. + +--- + +## 4. Authority & tenancy integration + +1. Create Authority clients for each backend (`observability-prometheus`, `observability-loki`, `observability-tempo`). + ```bash + stella authority client create observability-prometheus \ + --scopes obs:read \ + --audience observability --description "Prometheus collector scrape" + stella authority client create observability-loki \ + --scopes obs:logs timeline:read \ + --audience observability --description "Loki ingestion" + stella authority client create observability-tempo \ + --scopes obs:traces \ + --audience observability --description "Tempo ingestion" + ``` +2. Mint tokens/credentials and store them in the secrets above (see staging bootstrap commands). Example: + ```bash + stella authority token issue observability-prometheus --ttl 30d > prometheus-stage.token + ``` +3. Update ingress/gateway policies to forward `X-StellaOps-Tenant` into Loki/Tempo so tenant headers propagate end-to-end, and ensure each workload sets `tenant.id` attributes (see `docs/observability/observability.md`). + +--- + +## 5. Retention & isolation + +- Adjust `deploy/telemetry/storage/tenants/*.yaml` to set per-tenant retention and ingestion limits. +- Configure object storage (S3, GCS, Azure Blob) when moving beyond filesystem storage. +- For air-gapped deployments, mirror the telemetry bundle using `ops/devops/telemetry/package_offline_bundle.py` and import inside the Offline Kit staging directory. + +--- + +## 6. Operational checklist + +- [ ] Certificates rotated and secrets updated. +- [ ] Prometheus scrape succeeds (`curl -sk --cert client.crt --key client.key https://collector:9464`). +- [ ] Tempo and Loki report tenant activity (`/api/status`). +- [ ] Retention policy tested by uploading sample data and verifying expiry. +- [ ] Alerts wired into SLO evaluator (DEVOPS-OBS-51-001). + +--- + +## 7. References + +- `deploy/telemetry/storage/README.md` +- `deploy/compose/docker-compose.telemetry-storage.yaml` +- `docs/ops/telemetry-collector.md` +- `docs/observability/observability.md` diff --git a/docs/ops/zastava-runtime-operations.md b/docs/ops/zastava-runtime-operations.md index 9c5c9b0d..c437a3f2 100644 --- a/docs/ops/zastava-runtime-operations.md +++ b/docs/ops/zastava-runtime-operations.md @@ -137,19 +137,33 @@ Runtime events emitted by Observer now include `process.buildId` (from the ELF `buildIds` list per digest. Operators can use these hashes to locate debug artifacts during incident response: -1. Capture the hash from CLI/webhook/Scanner API (example: +1. Capture the hash from CLI/webhook/Scanner API—for example: + ```bash + stellaops-cli runtime policy test --image --namespace + ``` + Copy one of the `Build IDs` (e.g. `5f0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789`). -2. Derive the path: `/` under the debug store, e.g. - `/var/opt/debug/.build-id/5f/0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789.debug`. +2. Derive the debug path (`/` under `.build-id`) and check it exists: + ```bash + ls /var/opt/debug/.build-id/5f/0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789.debug + ``` 3. If the file is missing, rehydrate it from Offline Kit bundles or the - `debug-store` object bucket (mirror of release artefacts). Use: - ```sh + `debug-store` object bucket (mirror of release artefacts): + ```bash oras cp oci://registry.internal/debug-store:latest . --include \ "5f/0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789.debug" ``` -4. Attach the `.debug` file in `gdb`/`lldb` or feed it to `eu-unstrip` when - preparing symbolized traces. -5. For musl-based images, expect shorter build-id footprints. Missing hashes in +4. Confirm the running process advertises the same GNU build-id before + symbolising: + ```bash + readelf -n /proc/$(pgrep -f payments-api | head -n1)/exe | grep -i 'Build ID' + ``` +5. Attach the `.debug` file in `gdb`/`lldb`, feed it to `eu-unstrip`, or cache it + in `debuginfod` for fleet-wide symbol resolution: + ```bash + debuginfod-find debuginfo 5f0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789 >/tmp/payments-api.debug + ``` +6. For musl-based images, expect shorter build-id footprints. Missing hashes in runtime events indicate stripped binaries without the GNU note—schedule a rebuild with `-Wl,--build-id` enabled or add the binary to the debug-store allowlist so the scanner can surface a fallback symbol package. diff --git a/docs/policy/dsl.md b/docs/policy/dsl.md new file mode 100644 index 00000000..7d541807 --- /dev/null +++ b/docs/policy/dsl.md @@ -0,0 +1,294 @@ +# Stella Policy DSL (`stella-dsl@1`) + +> **Audience:** Policy authors, reviewers, and tooling engineers building lint/compile flows for the Policy Engine v2 rollout (Sprint 20). + +This document specifies the `stella-dsl@1` grammar, semantics, and guardrails used by Stella Ops to transform SBOM facts, Concelier advisories, and Excititor VEX statements into effective findings. Use it with the [Policy Engine Overview](overview.md) for architectural context and the upcoming lifecycle/run guides for operational workflows. + +--- + +## 1 · Design Goals + +- **Deterministic:** Same policy + same inputs ⇒ identical findings on every machine. +- **Declarative:** No arbitrary loops, network calls, or clock access. +- **Explainable:** Every decision records the rule, inputs, and rationale in the explain trace. +- **Lean authoring:** Common precedence, severity, and suppression patterns are first-class. +- **Offline-friendly:** Grammar and built-ins avoid cloud dependencies, run the same in sealed deployments. + +--- + +## 2 · Document Structure + +Policy packs ship one or more `.stella` files. Each file contains exactly one `policy` block: + +```dsl +policy "Default Org Policy" syntax "stella-dsl@1" { + metadata { + description = "Baseline severity + VEX precedence" + tags = ["baseline","vex"] + } + + profile severity { + map vendor_weight { + source "GHSA" => +0.5 + source "OSV" => +0.0 + source "VendorX" => -0.2 + } + env exposure_adjustments { + if env.runtime == "serverless" then -0.5 + if env.exposure == "internal-only" then -1.0 + } + } + + rule vex_precedence priority 10 { + when vex.any(status in ["not_affected","fixed"]) + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + because "Strong vendor justification prevails"; + } +} +``` + +High-level layout: + +| Section | Purpose | +|---------|---------| +| `metadata` | Optional descriptive fields surfaced in Console/CLI. | +| `imports` | Reserved for future reuse (not yet implemented in `@1`). | +| `profile` blocks | Declarative scoring modifiers (`severity`, `trust`, `reachability`). | +| `rule` blocks | When/then logic applied to each `(component, advisory, vex[])` tuple. | +| `settings` | Optional evaluation toggles (sampling, default status overrides). | + +--- + +## 3 · Lexical Rules + +- **Case sensitivity:** Keywords are lowercase; identifiers are case-sensitive. +- **Whitespace:** Space, tab, newline act as separators. Indentation is cosmetic. +- **Comments:** `// inline` and `/* block */` are ignored. +- **Literals:** + - Strings use double quotes (`"text"`); escape with `\"`, `\n`, `\t`. + - Numbers are decimal; suffix `%` allowed for percentage weights (`-2.5%` becomes `-0.025`). + - Booleans: `true`, `false`. + - Lists: `[1, 2, 3]`, `["a","b"]`. +- **Identifiers:** Start with letter or underscore, continue with letters, digits, `_`. +- **Operators:** `=`, `==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not in`, `and`, `or`, `not`, `:=`. + +--- + +## 4 · Grammar (EBNF) + +```ebnf +policy = "policy", string, "syntax", string, "{", policy-body, "}" ; +policy-body = { metadata | profile | settings | rule | helper } ; + +metadata = "metadata", "{", { meta-entry }, "}" ; +meta-entry = identifier, "=", (string | list) ; + +profile = "profile", identifier, "{", { profile-item }, "}" ; +profile-item= map | env-map | scalar ; +map = "map", identifier, "{", { "source", string, "=>", number, ";" }, "}" ; +env-map = "env", identifier, "{", { "if", expression, "then", number, ";" }, "}" ; +scalar = identifier, "=", (number | string | list), ";" ; + +settings = "settings", "{", { setting-entry }, "}" ; +setting-entry = identifier, "=", (number | string | boolean), ";" ; + +rule = "rule", identifier, [ "priority", integer ], "{", + "when", predicate, + { "and", predicate }, + "then", { action }, + [ "else", { action } ], + [ "because", string ], + "}" ; + +predicate = expression ; +expression = term, { ("and" | "or"), term } ; +term = ["not"], factor ; +factor = comparison | membership | function-call | literal | identifier | "(" expression ")" ; +comparison = value, comparator, value ; +membership = value, ("in" | "not in"), list ; +value = identifier | literal | function-call | field-access ; +field-access= identifier, { ".", identifier | "[" literal "]" } ; +function-call = identifier, "(", [ arg-list ], ")" ; +arg-list = expression, { ",", expression } ; +literal = string | number | boolean | list ; + +action = assignment | ignore | escalate | require | warn | defer | annotate ; +assignment = target, ":=", expression, ";" ; +target = identifier, { ".", identifier } ; +ignore = "ignore", [ "until", expression ], [ "because", string ], ";" ; +escalate = "escalate", [ "to", expression ], [ "when", expression ], ";" ; +require = "requireVex", "{", require-fields, "}", ";" ; +warn = "warn", [ "message", string ], ";" ; +defer = "defer", [ "until", expression ], ";" ; +annotate = "annotate", identifier, ":=", expression, ";" ; +``` + +Notes: + +- `helper` is reserved for shared calculcations (not yet implemented in `@1`). +- `else` branch executes only if `when` predicates evaluate truthy **and** no prior rule earlier in priority handled the tuple. +- Semicolons inside rule bodies are optional when each clause is on its own line; the compiler emits canonical semicolons in IR. + +--- + +## 5 · Evaluation Context + +Within predicates and actions you may reference the following namespaces: + +| Namespace | Fields | Description | +|-----------|--------|-------------| +| `sbom` | `purl`, `name`, `version`, `licenses`, `layerDigest`, `tags`, `usedByEntrypoint` | Component metadata from Scanner. | +| `advisory` | `id`, `source`, `aliases`, `severity`, `cvss`, `publishedAt`, `modifiedAt`, `content.raw` | Canonical Concelier advisory view. | +| `vex` | `status`, `justification`, `statementId`, `timestamp`, `scope` | Current VEX statement when iterating; aggregator helpers available. | +| `vex.any(...)`, `vex.all(...)`, `vex.count(...)` | Functions operating over all matching statements. | +| `run` | `policyId`, `policyVersion`, `tenant`, `timestamp` | Metadata for explain annotations. | +| `env` | Arbitrary key/value pairs injected per run (e.g., `environment`, `runtime`). | +| `telemetry` | Optional reachability signals; missing fields evaluate to `unknown`. | +| `profile.` | Values computed inside profile blocks (maps, scalars). | + +Missing fields evaluate to `null`, which is falsey in boolean context and propagates through comparisons unless explicitly checked. + +--- + +## 6 · Built-ins (v1) + +| Function / Property | Signature | Description | +|---------------------|-----------|-------------| +| `normalize_cvss(advisory)` | `Advisory → SeverityScalar` | Parses `advisory.content.raw` for CVSS data; falls back to policy maps. | +| `cvss(score, vector)` | `double × string → SeverityScalar` | Constructs a severity object manually. | +| `severity_band(value)` | `string → SeverityBand` | Normalises strings like `"critical"`, `"medium"`. | +| `risk_score(base, modifiers...)` | Variadic | Multiplies numeric modifiers (severity × trust × reachability). | +| `vex.any(predicate)` | `(Statement → bool) → bool` | `true` if any statement satisfies predicate. | +| `vex.all(predicate)` | `(Statement → bool) → bool` | `true` if all statements satisfy predicate. | +| `vex.latest()` | `→ Statement` | Lexicographically newest statement. | +| `advisory.has_tag(tag)` | `string → bool` | Checks advisory metadata tags. | +| `advisory.matches(pattern)` | `string → bool` | Glob match against advisory identifiers. | +| `sbom.has_tag(tag)` | `string → bool` | Uses SBOM inventory tags (usage vs inventory). | +| `exists(expression)` | `→ bool` | `true` when value is non-null/empty. | +| `coalesce(a, b, ...)` | `→ value` | First non-null argument. | +| `days_between(dateA, dateB)` | `→ int` | Absolute day difference (UTC). | +| `percent_of(part, whole)` | `→ double` | Fractions for scoring adjustments. | +| `lowercase(text)` | `string → string` | Normalises casing deterministically (InvariantCulture). | + +All built-ins are pure; if inputs are null the result is null unless otherwise noted. + +--- + +## 7 · Rule Semantics + +1. **Ordering:** Rules execute in ascending `priority`. When priorities tie, lexical order defines precedence. +2. **Short-circuit:** Once a rule sets `status`, subsequent rules only execute if they use `combine`. Use this sparingly to avoid ambiguity. +3. **Actions:** + - `status := ` – Allowed values: `affected`, `not_affected`, `fixed`, `suppressed`, `under_investigation`, `escalated`. + - `severity := ` – Either from `normalize_cvss`, `cvss`, or numeric map; ensures `normalized` and `score`. + - `ignore until ` – Temporarily treats finding as suppressed until timestamp; recorded in explain trace. + - `warn message ""` – Adds warn verdict and deducts `warnPenalty`. + - `escalate to severity_band("critical") when condition` – Forces verdict severity upward when condition true. + - `requireVex { vendors = ["VendorX"], justifications = ["component_not_present"] }` – Fails evaluation if matching VEX evidence absent. + - `annotate reason := "text"` – Adds free-form key/value pairs to explain payload. +4. **Because clause:** Mandatory for actions changing status or severity; captured verbatim in explain traces. + +--- + +## 8 · Scoping Helpers + +- **Maps:** Use `profile severity { map vendor_weight { ... } }` to declare additive factors. Retrieve with `profile.severity.vendor_weight["GHSA"]`. +- **Environment overrides:** `env` profiles allow conditional adjustments based on runtime metadata. +- **Tenancy:** `run.tenant` ensures policies remain tenant-aware; avoid hardcoding single-tenant IDs. +- **Default values:** Use `settings { default_status = "affected"; }` to override built-in defaults. + +--- + +## 9 · Examples + +### 9.1 Baseline Severity Normalisation + +```dsl +rule advisory_normalization { + when advisory.source in ["GHSA","OSV"] + then severity := normalize_cvss(advisory) + because "Align vendor severity to CVSS baseline"; +} +``` + +### 9.2 VEX Override with Quiet Mode + +```dsl +rule vex_strong_claim priority 5 { + when vex.any(status == "not_affected") + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + annotate winning_statement := vex.latest().statementId + warn message "VEX override applied" + because "Strong VEX justification"; +} +``` + +### 9.3 Environment-Specific Escalation + +```dsl +rule internet_exposed_guard { + when env.exposure == "internet" + and severity.normalized >= "High" + then escalate to severity_band("Critical") + because "Internet-exposed assets require critical posture"; +} +``` + +### 9.4 Anti-pattern (flagged by linter) + +```dsl +rule catch_all { + when true + then status := "suppressed" + because "Suppress everything" // ❌ Fails lint: unbounded suppression +} +``` + +--- + +## 10 · Validation & Tooling + +- `stella policy lint` ensures: + - Grammar compliance and canonical formatting. + - Static determinism guard (no forbidden namespaces). + - Anti-pattern detection (e.g., unconditional suppression, missing `because`). +- `stella policy compile` emits IR (`.stella.ir.json`) and SHA-256 digest used in `policy_runs`. +- CI pipelines (see `DEVOPS-POLICY-20-001`) compile sample packs and fail on lint violations. +- Simulation harnesses (`stella policy simulate`) highlight provided/queried fields so policy authors affirm assumptions before promotion. + +--- + +## 11 · Anti-patterns & Mitigations + +| Anti-pattern | Risk | Mitigation | +|--------------|------|------------| +| Catch-all suppress/ignore without scope | Masks all findings | Linter blocks rules with `when true` unless `priority` > 1000 and justification includes remediation plan. | +| Comparing strings with inconsistent casing | Missed matches | Wrap comparisons in `lowercase(value)` to align casing or normalise metadata during ingest. | +| Referencing `telemetry` without fallback | Null propagation | Wrap access in `exists(telemetry.reachability)`. | +| Hardcoding tenant IDs | Breaks multi-tenant | Prefer `env.tenantTag` or metadata-sourced predicates. | +| Duplicated rule names | Explain trace ambiguity | Compiler enforces unique `rule` identifiers within a policy. | + +--- + +## 12 · Versioning & Compatibility + +- `syntax "stella-dsl@1"` is mandatory. +- Future revisions (`@2`, …) will be additive; existing packs continue to compile with their declared version. +- The compiler canonicalises documents (sorted keys, normalised whitespace) before hashing to ensure reproducibility. + +--- + +## 13 · Compliance Checklist + +- [ ] **Grammar validated:** Policy compiles with `stella policy lint` and matches `syntax "stella-dsl@1"`. +- [ ] **Deterministic constructs only:** No use of forbidden namespaces (`DateTime.Now`, `Guid.NewGuid`, external services). +- [ ] **Rationales present:** Every status/severity change includes a `because` clause or `annotate` entry. +- [ ] **Scoped suppressions:** Rules that ignore/suppress findings reference explicit components, vendors, or VEX justifications. +- [ ] **Explain fields verified:** `annotate` keys align with Console/CLI expectations (documented in upcoming lifecycle guide). +- [ ] **Offline parity tested:** Policy pack simulated in sealed mode (`--sealed`) to confirm absence of network dependencies. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* diff --git a/docs/policy/lifecycle.md b/docs/policy/lifecycle.md new file mode 100644 index 00000000..4f4c2695 --- /dev/null +++ b/docs/policy/lifecycle.md @@ -0,0 +1,239 @@ +# Policy Lifecycle & Approvals + +> **Audience:** Policy authors, reviewers, security approvers, release engineers. +> **Scope:** End-to-end flow for `stella-dsl@1` policies from draft through archival, including CLI/Console touch-points, Authority scopes, audit artefacts, and offline considerations. + +This guide explains how a policy progresses through Stella Ops, which roles are involved, and the artefacts produced at every step. Pair it with the [Policy Engine Overview](overview.md), [DSL reference](dsl.md), and upcoming run documentation to ensure consistent authoring and rollout. + +--- + +## 1 · Protocol Summary + +- Policies are **immutable versions** attached to a stable `policy_id`. +- Lifecycle states: `draft → submitted → approved → active → archived`. +- Every transition requires explicit Authority scopes and produces structured events + storage artefacts (`policies`, `policy_runs`, audit log collections). +- Simulation and CI gating happen **before** approvals can be granted. +- Activation triggers (runs, bundle exports, CLI `promote`) operate on the **latest approved** version per tenant. + +```mermaid +stateDiagram-v2 + [*] --> Draft + Draft --> Draft: edit/save (policy:write) + Draft --> Submitted: submit(reviewers) (policy:submit) + Submitted --> Draft: requestChanges (policy:write) + Submitted --> Approved: approve (policy:approve) + Approved --> Active: activate/run (policy:run) + Active --> Archived: archive (policy:archive) + Approved --> Archived: superseded/explicit archive + Archived --> [*] +``` + +--- + +## 2 · Roles & Authority Scopes + +| Role (suggested) | Required scopes | Responsibilities | +|------------------|-----------------|------------------| +| **Policy Author** | `policy:write`, `policy:submit`, `policy:simulate` | Draft DSL, run local/CI simulations, submit for review. | +| **Policy Reviewer** | `policy:review`, `policy:simulate`, `policy:runs` | Comment on submissions, demand additional simulations, request changes. | +| **Policy Approver** | `policy:approve`, `policy:runs`, `policy:audit` | Grant final approval, ensure sign-off evidence captured. | +| **Policy Operator** | `policy:run`, `policy:activate`, `findings:read` | Trigger full/incremental runs, monitor results, roll back to previous version. | +| **Policy Auditor** | `policy:audit`, `findings:read`, `policy:history` | Review past versions, verify attestations, respond to compliance requests. | +| **Policy Engine Service** | `effective:write`, `findings:read` | Materialise effective findings during runs; no approval capabilities. | + +> Scopes are issued by Authority (`AUTH-POLICY-20-001`). Tenants may map organisational roles (e.g., `secops.approver`) to these scopes via issuer policy. + +--- + +## 3 · Lifecycle Stages in Detail + +### 3.1 Draft + +- **Who:** Authors (policy:write). +- **Tools:** Console editor, `stella policy edit`, policy DSL files. +- **Actions:** + - Author DSL leveraging [stella-dsl@1](dsl.md). + - Run `stella policy lint` and `stella policy simulate --sbom ` locally. + - Attach rationale metadata (`metadata.description`, tags). +- **Artefacts:** + - `policies` document with `status=draft`, `version=n`, `provenance.created_by`. + - Local IR cache (`.stella.ir.json`) generated by CLI compile. +- **Guards:** + - Draft versions never run in production. + - CI must lint drafts before allowing submission PRs (see `DEVOPS-POLICY-20-001`). + +### 3.2 Submission + +- **Who:** Authors with `policy:submit`. +- **Tools:** Console “Submit for review” button, `stella policy submit --reviewers ...`. +- **Actions:** + - Provide review notes and required simulations (CLI uploads attachments). + - Choose reviewer groups; Authority records them in submission metadata. +- **Artefacts:** + - Policy document transitions to `status=submitted`, capturing `submitted_by`, `submitted_at`, reviewer list, simulation digest references. + - Audit event `policy.submitted` (Authority timeline / Notifier integration). +- **Guards:** + - Submission blocked unless latest lint + compile succeed (<24 h freshness). + - Must reference at least one simulation artefact (CLI enforces via `--attach`). + +### 3.3 Review (Submitted) + +- **Who:** Reviewers (`policy:review`), optionally authors responding. +- **Tools:** Console review pane (line comments, overall verdict), `stella policy review`. +- **Actions:** + - Inspect DSL diff vs previous approved version. + - Run additional `simulate` jobs (UI button or CLI). + - Request changes → policy returns to `draft` with comment log. +- **Artefacts:** + - Comments stored in `policy_reviews` collection with timestamps, resolved flag. + - Additional simulation run records appended to submission metadata. +- **Guards:** + - Approval cannot proceed until all blocking comments resolved. + - Required reviewers (Authority rule) must vote before approver sees “Approve” button. + +### 3.4 Approval + +- **Who:** Approvers (`policy:approve`). +- **Tools:** Console “Approve”, CLI `stella policy approve --version n --note "rationale"`. +- **Actions:** + - Confirm compliance checks (see §6) all green. + - Provide approval note (mandatory string captured in audit trail). +- **Artefacts:** + - Policy `status=approved`, `approved_by`, `approved_at`, `approval_note`. + - Audit event `policy.approved` plus optional Notifier broadcast. + - Immutable approval record stored in `policy_history`. +- **Guards:** + - Approver cannot be same identity as author (enforced by Authority config). + - Approver must attest to successful simulation diff review (`--attach diff.json`). + +### 3.5 Activation & Runs + +- **Who:** Operators (`policy:run`, `policy:activate`). +- **Tools:** Console “Promote to active”, CLI `stella policy activate --version n`, `stella policy run`. +- **Actions:** + - Mark approved version as tenant’s active policy. + - Trigger full run or rely on orchestrator for incremental runs. + - Monitor results via Console dashboards or CLI run logs. +- **Artefacts:** + - `policy_runs` entries with `mode=full|incremental`, `policy_version=n`. + - Effective findings collections updated; explain traces stored. + - Activation event `policy.activated` with `runId`. +- **Guards:** + - Activation blocked if previous full run <24 h old failed or is pending. + - Selection of SBOM/advisory snapshots uses consistent cursors recorded for reproducibility. + +### 3.6 Archival / Rollback + +- **Who:** Approvers or Operators with `policy:archive`. +- **Tools:** Console menu, CLI `stella policy archive --version n --reason`. +- **Actions:** + - Retire policies superseded by newer versions or revert to older approved version (`stella policy activate --version n-1`). + - Export archived version for audit bundles (Offline Kit integration). +- **Artefacts:** + - Policy `status=archived`, `archived_by`, `archived_at`, reason. + - Audit event `policy.archived`. + - Exported DSSE-signed policy pack stored if requested. +- **Guards:** + - Archival cannot proceed while runs using that version are in-flight. + - Rollback requires documented incident reference. + +--- + +## 4 · Tooling Touchpoints + +| Stage | Console | CLI | API | +|-------|---------|-----|-----| +| Draft | Inline linting, simulation panel | `stella policy lint`, `edit`, `simulate` | `POST /policies`, `PUT /policies/{id}/versions/{v}` | +| Submit | Submit modal (attach simulations) | `stella policy submit` | `POST /policies/{id}/submit` | +| Review | Comment threads, diff viewer | `stella policy review --approve/--request-changes` | `POST /policies/{id}/reviews` | +| Approve | Approve dialog | `stella policy approve` | `POST /policies/{id}/approve` | +| Activate | Promote button, run scheduler | `stella policy activate`, `run`, `simulate` | `POST /policies/{id}/run`, `POST /policies/{id}/activate` | +| Archive | Archive / rollback menu | `stella policy archive` | `POST /policies/{id}/archive` | + +All CLI commands emit structured JSON by default; use `--format table` for human review. + +--- + +## 5 · Audit & Observability + +- **Storage:** + - `policies` retains all versions with provenance metadata. + - `policy_reviews` stores reviewer comments, timestamps, attachments. + - `policy_history` summarises transitions (state, actor, note, diff digest). + - `policy_runs` retains input cursors and determinism hash per run. +- **Events:** + - `policy.submitted`, `policy.review.requested`, `policy.approved`, `policy.activated`, `policy.archived`, `policy.rollback`. + - Routed to Notifier + Timeline Indexer; offline deployments log to local event store. +- **Logs & metrics:** + - Policy Engine logs include `policyId`, `policyVersion`, `runId`, `approvalNote`. + - Observability dashboards (see forthcoming `/docs/observability/policy.md`) highlight pending approvals, run SLA, VEX overrides. +- **Reproducibility:** + - Each state transition stores IR checksum and simulation diff digests, enabling offline audit replay. + +--- + +## 6 · Compliance Gates + +| Gate | Stage | Enforced by | Requirement | +|------|-------|-------------|-------------| +| **DSL lint** | Draft → Submit | CLI/CI | `stella policy lint` successful within 24 h. | +| **Simulation evidence** | Submit | CLI/Console | Attach diff from `stella policy simulate` covering baseline SBOM set. | +| **Reviewer quorum** | Submit → Approve | Authority | Minimum approver/reviewer count configurable per tenant. | +| **Determinism CI** | Approve | DevOps job | Twin run diff passes (`DEVOPS-POLICY-20-003`). | +| **Activation health** | Approve → Activate | Policy Engine | Last run status succeeded; orchestrator queue healthy. | +| **Export validation** | Archive | Offline Kit | DSSE-signed policy pack generated for long-term retention. | + +Failure of any gate emits a `policy.lifecycle.violation` event and blocks transition until resolved. + +--- + +## 7 · Offline / Air-Gap Considerations + +- Offline Kit bundles include: + - Approved policy packs (`.policy.bundle` + DSSE signatures). + - Submission/approval audit logs. + - Simulation diff JSON for reproducibility. +- Air-gapped sites operate with the same lifecycle: + - Approvals happen locally; Authority runs in enclave. + - Rollout requires manual import of policy packs from connected environment via signed bundles. + - `stella policy simulate --sealed` ensures no outbound calls; required before approval in sealed mode. + +--- + +## 8 · Incident Response & Rollback + +- Incident mode (triggered via `policy incident activate`) forces: + - Immediate incremental run to evaluate mitigation policies. + - Expanded trace retention for affected runs. + - Automatic snapshot of currently active policies for evidence locker. +- Rollback path: + 1. `stella policy activate --version ` with incident note. + 2. Orchestrator schedules full run to ensure findings align. + 3. Archive problematic version with reason referencing incident ticket. +- Post-incident review must confirm new version passes gates before re-activation. + +--- + +## 9 · CI/CD Integration (Reference) + +- **Pre-merge:** run lint + simulation jobs against golden SBOM fixtures. +- **Post-merge (main):** compile, compute IR checksum, stage for Offline Kit. +- **Nightly:** determinism replay, `policy simulate` diff drift alerts, backlog of pending approvals. +- **Notifications:** Slack/Email via Notifier when submissions await review > SLA or approvals succeed. + +--- + +## 10 · Compliance Checklist + +- [ ] **Role mapping validated:** Authority issuer config maps organisational roles to required `policy:*` scopes (per tenant). +- [ ] **Submission evidence attached:** Latest simulation diff and lint artefacts linked to submission. +- [ ] **Reviewer quorum met:** All required reviewers approved or acknowledged; no unresolved blocking comments. +- [ ] **Approval note logged:** Approver justification recorded in audit trail alongside IR checksum. +- [ ] **Activation guard passed:** Latest run status success, orchestrator queue healthy, determinism job green. +- [ ] **Archive bundles produced:** When archiving, DSSE-signed policy pack exported and stored for offline retention. +- [ ] **Offline parity proven:** For sealed deployments, `--sealed` simulations executed and logged before approval. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* + diff --git a/docs/policy/overview.md b/docs/policy/overview.md new file mode 100644 index 00000000..d94384b8 --- /dev/null +++ b/docs/policy/overview.md @@ -0,0 +1,173 @@ +# Policy Engine Overview + +> **Goal:** Evaluate organisation policies deterministically against scanner SBOMs, Concelier advisories, and Excititor VEX evidence, then publish effective findings that downstream services can trust. + +This document introduces the v2 Policy Engine: how the service fits into Stella Ops, the artefacts it produces, the contracts it honours, and the guardrails that keep policy decisions reproducible across air-gapped and connected deployments. + +--- + +## 1 · Role in the Platform + +- **Purpose:** Compose policy verdicts by reconciling SBOM inventory, advisory metadata, VEX statements, and organisation rules. +- **Form factor:** Dedicated `.NET 10` Minimal API host (`StellaOps.Policy.Engine`) plus worker orchestration. Policies are defined in `stella-dsl@1` packs compiled to an intermediate representation (IR) with a stable SHA-256 digest. +- **Tenancy:** All workloads run under Authority-enforced scopes (`policy:*`, `findings:read`, `effective:write`). Only the Policy Engine identity may materialise effective findings collections. +- **Consumption:** Findings ledger, Console, CLI, and Notify read the published `effective_finding_{policyId}` materialisations and policy run ledger (`policy_runs`). +- **Offline parity:** Bundled policies import/export alongside advisories and VEX. In sealed mode the engine degrades gracefully, annotating explanations whenever cached signals replace live lookups. + +--- + +## 2 · High-Level Architecture + +```mermaid +flowchart LR + subgraph Inputs + A[Scanner SBOMs
Inventory & Usage] + B[Concelier Advisories
Canonical linksets] + C[Excititor VEX
Consensus status] + D[Policy Packs
stella-dsl@1] + end + subgraph PolicyEngine["StellaOps.Policy.Engine"] + P1[DSL Compiler
IR + Digest] + P2[Joiners
SBOM ↔ Advisory ↔ VEX] + P3[Deterministic Evaluator
Rule hits + scoring] + P4[Materialisers
effective findings] + P5[Run Orchestrator
Full & incremental] + end + subgraph Outputs + O1[Effective Findings Collections] + O2[Explain Traces
Rule hit lineage] + O3[Metrics & Traces
policy_run_seconds,
rules_fired_total] + O4[Simulation/Preview Feeds
CLI & Studio] + end + + A --> P2 + B --> P2 + C --> P2 + D --> P1 --> P3 + P2 --> P3 --> P4 --> O1 + P3 --> O2 + P5 --> P3 + P3 --> O3 + P3 --> O4 +``` + +--- + +## 3 · Core Concepts + +| Concept | Description | +|---------|-------------| +| **Policy Pack** | Versioned bundle of DSL documents, metadata, and checksum manifest. Packs import/export via CLI and Offline Kit bundles. | +| **Policy Digest** | SHA-256 of the canonical IR; used for caching, explain trace attribution, and audit proofs. | +| **Effective Findings** | Append-only Mongo collections (`effective_finding_{policyId}`) storing the latest verdict per finding, plus history sidecars. | +| **Policy Run** | Execution record persisted in `policy_runs` capturing inputs, run mode, timings, and determinism hash. | +| **Explain Trace** | Structured tree showing rule matches, data provenance, and scoring components for UI/CLI explain features. | +| **Simulation** | Dry-run evaluation that compares a candidate pack against the active pack and produces verdict diffs without persisting results. | +| **Incident Mode** | Elevated sampling/trace capture toggled automatically when SLOs breach; emits events for Notifier and Timeline Indexer. | + +--- + +## 4 · Inputs & Pre-processing + +### 4.1 SBOM Inventory + +- **Source:** Scanner.WebService publishes inventory/usage SBOMs plus BOM-Index (roaring bitmap) metadata. +- **Consumption:** Policy joiners use the index to expand candidate components quickly, keeping evaluation under the `< 5 s` warm path budget. +- **Schema:** CycloneDX Protobuf + JSON views; Policy Engine reads canonical projections via shared SBOM adapters. + +### 4.2 Advisory Corpus + +- **Source:** Concelier exports canonical advisories with deterministic identifiers, linksets, and equivalence tables. +- **Contract:** Policy Engine only consumes raw `content.raw`, `identifiers`, and `linkset` fields per Aggregation-Only Contract (AOC); derived precedence remains a policy concern. + +### 4.3 VEX Evidence + +- **Source:** Excititor consensus service resolves OpenVEX / CSAF statements, preserving conflicts. +- **Usage:** Policy rules can require specific VEX vendors or justification codes; evaluator records when cached evidence substitutes for live statements (sealed mode). + +### 4.4 Policy Packs + +- Authored in Policy Studio or CLI, validated against the `stella-dsl@1` schema. +- Compiler performs canonicalisation (ordering, defaulting) before emitting IR and digest. +- Packs bundle scoring profiles, allowlist metadata, and optional reachability weighting tables. + +--- + +## 5 · Evaluation Flow + +1. **Run selection** – Orchestrator accepts `full`, `incremental`, or `simulate` jobs. Incremental runs listen to change streams from Concelier, Excititor, and SBOM imports to scope re-evaluation. +2. **Input staging** – Candidates fetched in deterministic batches; identity graph from Concelier strengthens PURL lookups. +3. **Rule execution** – Evaluator walks rules in lexical order (first-match wins). Actions available: `block`, `ignore`, `warn`, `defer`, `escalate`, `requireVex`, each supporting quieting semantics where permitted. +4. **Scoring** – `PolicyScoringConfig` applies severity, trust, reachability weights plus penalties (`warnPenalty`, `ignorePenalty`, `quietPenalty`). +5. **Verdict and explain** – Engine constructs `PolicyVerdict` records with inputs, quiet flags, unknown confidence bands, and provenance markers; explain trees capture rule lineage. +6. **Materialisation** – Effective findings collections are upserted append-only, stamped with run identifier, policy digest, and tenant. +7. **Publishing** – Completed run writes to `policy_runs`, emits metrics (`policy_run_seconds`, `rules_fired_total`, `vex_overrides_total`), and raises events for Console/Notify subscribers. + +--- + +## 6 · Run Modes + +| Mode | Trigger | Scope | Persistence | Typical Use | +|------|---------|-------|-------------|-------------| +| **Full** | Manual CLI (`stella policy run`), scheduled nightly, or emergency rebaseline | Entire tenant | Writes effective findings and run record | After policy publish or major advisory/VEX import | +| **Incremental** | Change-stream queue driven by Concelier/Excititor/SBOM deltas | Only affected artefacts | Writes effective findings and run record | Continuous upkeep; ensures SLA ≤ 5 min from source change | +| **Simulate** | CLI/Studio preview, CI pipelines | Candidate subset (diff against baseline) | No materialisation; produces explain & diff payloads | Policy authoring, CI regression suites | + +All modes are cancellation-aware and checkpoint progress for replay in case of deployment restarts. + +--- + +## 7 · Outputs & Integrations + +- **APIs** – Minimal API exposes policy CRUD, run orchestration, explain fetches, and cursor-based listing of effective findings (see `/docs/api/policy.md` once published). +- **CLI** – `stella policy simulate/run/show` commands surface JSON verdicts, exit codes, and diff summaries suitable for CI gating. +- **Console / Policy Studio** – UI reads explain traces, policy metadata, approval workflow status, and simulation diffs to guide reviewers. +- **Findings Ledger** – Effective findings feed downstream export, Notify, and risk scoring jobs. +- **Air-gap bundles** – Offline Kit includes policy packs, scoring configs, and explain indexes; export commands generate DSSE-signed bundles for transfer. + +--- + +## 8 · Determinism & Guardrails + +- **Deterministic inputs** – All joins rely on canonical linksets and equivalence tables; batches are sorted, and random/wall-clock APIs are blocked by static analysis plus runtime guards (`ERR_POL_004`). +- **Stable outputs** – Canonical JSON serializers sort keys; digests recorded in run metadata enable reproducible diffs across machines. +- **Idempotent writes** – Materialisers upsert using `{policyId, findingId, tenant}` keys and retain prior versions with append-only history. +- **Sandboxing** – Policy evaluation executes in-process with timeouts; restart-only plug-ins guarantee no runtime DLL injection. +- **Compliance proof** – Every run stores digest of inputs (policy, SBOM batch, advisory snapshot) so auditors can replay decisions offline. + +--- + +## 9 · Security, Tenancy & Offline Notes + +- **Authority scopes:** Gateway enforces `policy:read`, `policy:write`, `policy:simulate`, `policy:runs`, `findings:read`, `effective:write`. Service identities must present DPoP-bound tokens. +- **Tenant isolation:** Collections partition by tenant identifier; cross-tenant queries require explicit admin scopes and return audit warnings. +- **Sealed mode:** In air-gapped deployments the engine surfaces `sealed=true` hints in explain traces, warning about cached EPSS/KEV data and suggesting bundle refreshes (see `docs/airgap/EPIC_16_AIRGAP_MODE.md` §3.7). +- **Observability:** Structured logs carry correlation IDs matching orchestrator job IDs; metrics integrate with OpenTelemetry exporters; sampled rule-hit logs redact policy secrets. +- **Incident response:** Incident mode can be forced via API, boosting trace retention and notifying Notifier through `policy.incident.activated` events. + +--- + +## 10 · Working with Policy Packs + +1. **Author** in Policy Studio or edit DSL files locally. Validate with `stella policy lint`. +2. **Simulate** against golden SBOM fixtures (`stella policy simulate --sbom fixtures/*.json`). Inspect explain traces for unexpected overrides. +3. **Publish** via API or CLI; Authority enforces review/approval workflows (`draft → review → approve → rollout`). +4. **Monitor** the subsequent incremental runs; if determinism diff fails in CI, roll back pack while investigating digests. +5. **Bundle** packs for offline sites with `stella policy bundle export` and distribute via Offline Kit. + +--- + +## 11 · Compliance Checklist + +- [ ] **Scopes enforced:** Confirm gateway policy requires `policy:*` and `effective:write` scopes for all mutating endpoints. +- [ ] **Determinism guard active:** Static analyzer blocks clock/RNG usage; CI determinism job diffing repeated runs passes. +- [ ] **Materialisation audit:** Effective findings collections use append-only writers and retain history per policy run. +- [ ] **Explain availability:** UI/CLI expose explain traces for every verdict; sealed-mode warnings display when cached evidence is used. +- [ ] **Offline parity:** Policy bundles (import/export) tested in sealed environment; air-gap degradations documented for operators. +- [ ] **Observability wired:** Metrics (`policy_run_seconds`, `rules_fired_total`, `vex_overrides_total`) and sampled rule hit logs emit to the shared telemetry pipeline with correlation IDs. +- [ ] **Documentation synced:** API (`/docs/api/policy.md`), DSL grammar (`/docs/policy/dsl.md`), lifecycle (`/docs/policy/lifecycle.md`), and run modes (`/docs/policy/runs.md`) cross-link back to this overview. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* + diff --git a/docs/policy/runs.md b/docs/policy/runs.md new file mode 100644 index 00000000..a2d74d26 --- /dev/null +++ b/docs/policy/runs.md @@ -0,0 +1,187 @@ +# Policy Runs & Orchestration + +> **Audience:** Policy Engine operators, Scheduler team, DevOps, and tooling engineers planning CI integrations. +> **Scope:** Run modes (`full`, `incremental`, `simulate`), orchestration pipeline, cursor management, replay/determinism guarantees, monitoring, and recovery procedures. + +Policies only generate value when they execute deterministically against current SBOM, advisory, and VEX inputs. This guide explains how runs are triggered, how the orchestrator scopes work, and what artefacts you should expect at each stage. + +--- + +## 1 · Run Modes at a Glance + +| Mode | Trigger sources | Scope | Persistence | Primary use | +|------|-----------------|-------|-------------|-------------| +| **Full** | Manual CLI (`stella policy run`), Console “Run now”, scheduled nightly job | Entire tenant (all registered SBOMs) | Writes `effective_finding_{policyId}` and `policy_runs` record | Baseline after policy approval, quarterly attestation, post-incident rechecks | +| **Incremental** | Change streams (Concelier advisories, Excititor VEX, SBOM imports), orchestrator cron | Only affected `(sbom, advisory)` tuples | Writes diffs to effective findings and run record | Continuous upkeep meeting ≤ 5 min SLA from input change | +| **Simulate** | Console review workspace, CLI (`stella policy simulate`), CI pipeline | Selected SBOM sample set (provided or golden set) | No materialisation; captures diff summary + explain traces | Authoring validation, regression safeguards, sealed-mode rehearsals | + +All modes record their status in `policy_runs` with deterministic metadata: + +```json +{ + "_id": "run:P-7:2025-10-26T14:05:11Z:3f9a", + "policy_id": "P-7", + "policy_version": 4, + "mode": "incremental", + "status": "succeeded", // queued | running | succeeded | failed | canceled | replay_pending + "inputs": { + "sbom_set": ["sbom:S-42","sbom:S-318"], + "advisory_cursor": "2025-10-26T13:59:00Z", + "vex_cursor": "2025-10-26T13:58:30Z", + "env": {"exposure":"internet"} + }, + "stats": { + "components": 1742, + "rules_fired": 68023, + "findings_written": 4321, + "vex_overrides": 210 + }, + "determinism_hash": "sha256:…", + "started_at": "2025-10-26T14:05:11Z", + "finished_at": "2025-10-26T14:06:01Z", + "tenant": "default" +} +``` + +> **Schemas & samples:** see `src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md` and the fixtures in `samples/api/scheduler/policy-*.json` for canonical payloads consumed by CLI/UI/worker integrations. + +--- + +## 2 · Pipeline Overview + +```mermaid +sequenceDiagram + autonumber + participant Trigger as Trigger (CLI / Console / Change Stream) + participant Orchestrator as Policy Orchestrator + participant Queue as Scheduler Queue (Mongo/NATS) + participant Engine as Policy Engine Workers + participant Concelier as Concelier Service + participant Excititor as Excititor Service + participant SBOM as SBOM Service + participant Store as Mongo (policy_runs & effective_finding_*) + participant Observability as Metrics/Events + + Trigger->>Orchestrator: Run request (mode, scope, env) + Orchestrator->>Queue: Enqueue PolicyRunRequest (idempotent key) + Queue->>Engine: Lease job (fairness window) + Engine->>Concelier: Fetch advisories + linksets (cursor-aware) + Engine->>Excititor: Fetch VEX statements (cursor-aware) + Engine->>SBOM: Fetch SBOM segments / BOM-Index + Engine->>Engine: Evaluate policy (deterministic batches) + Engine->>Store: Upsert effective findings + append history + Engine->>Store: Persist policy_runs record + determinism hash + Engine->>Observability: Emit metrics, traces, rule-hit logs + Engine->>Orchestrator: Ack completion / failure + Orchestrator->>Trigger: Notify (webhook, CLI, Console update) +``` + +- **Trigger** – CLI, Console, or automated change stream publishes a `PolicyRunRequest`. +- **Orchestrator** – Runs inside `StellaOps.Policy.Engine` worker host; applies fairness (tenant + policy quotas) and idempotency using run keys. +- **Queue** – Backed by Mongo + optional NATS for fan-out; supports leases and replay on crash. +- **Engine** – Stateless worker executing the deterministic evaluator. +- **Store** – Mongo collections: `policy_runs`, `effective_finding_{policyId}`, `policy_run_events` (append-only history), optional object storage for explain traces. +- **Observability** – Prometheus metrics (`policy_run_seconds`), OTLP traces, structured logs. + +--- + +## 3 · Input Scoping & Cursors + +### 3.1 Advisory & VEX Cursors + +- Each run records the latest Concelier change stream timestamp (`advisory_cursor`) and Excititor timestamp (`vex_cursor`). +- Incremental runs receive change batches `(feedId, lastOffset)`; orchestrator deduplicates using `change_digest`. +- Full runs set cursors to “current read time”, effectively resetting incremental baseline. + +### 3.2 SBOM Selection + +- Full runs enumerate all SBOM records declared active for the tenant. +- Incremental runs derive SBOM set by intersecting advisory/VEX changes with BOM-Index lookups (component → SBOM mapping). +- Simulations accept explicit SBOM list; if omitted, CLI uses `etc/policy/golden-sboms.json`. + +### 3.3 Environment Metadata + +- `env` block (free-form key/values) allows scenario-specific evaluation (e.g., `env.exposure=internet`). +- Stored verbatim in `policy_runs.inputs.env` for replay; orchestrator hashes environment data to avoid cache collisions. + +--- + +## 4 · Execution Semantics + +1. **Preparation:** Worker loads compiled IR for target policy version (cached by digest). +2. **Batching:** Candidate tuples are grouped by SBOM, then by advisory to maintain deterministic order; page size defaults to 1024 tuples. +3. **Evaluation:** Rules execute with first-match semantics; results captured as `PolicyVerdict`. +4. **Materialisation:** + - Upserts into `effective_finding_{policyId}` using `{policyId, sbomId, findingKey}`. + - Previous versions stored in `effective_finding_{policyId}_history`. +5. **Explain storage:** Full explain trees stored in blob store when `captureExplain=true`; incremental runs keep sampled traces (configurable). +6. **Completion:** Worker writes final status, stats, determinism hash (combination of policy digest + ordered input digests), and emits `policy.run.completed` event. + +--- + +## 5 · Retry, Replay & Determinism + +- **Retries:** Failures (network, validation) mark run `status=failed` and enqueue retry with exponential backoff capped at 3 attempts. Manual re-run via CLI resets counters. +- **Replay:** + - Use `policy_runs` record to assemble input snapshot (policy version, cursors, env). + - Fetch associated SBOM/advisory/VEX data via `stella policy replay --run ` which rehydrates data into a sealed bundle. + - Determinism hash mismatches between replay and recorded run indicate drift; CI job `DEVOPS-POLICY-20-003` compares successive runs to guard this. +- **Cancellation:** Manual `stella policy run cancel ` or orchestrator TTL triggers `status=canceled`; partial changes roll back via history append (no destructive delete). + +--- + +## 6 · Trigger Sources & Scheduling + +| Source | Description | SLAs | +|--------|-------------|------| +| **Nightly full run** | Default schedule per tenant; ensures baseline alignment. | Finish before 07:00 UTC | +| **Change stream** | Concelier (`advisory_raw`), Excititor (`vex_raw`), SBOM imports emit `policy.trigger.delta` events. | Start within 60 s; complete within 5 min | +| **Manual CLI/Console** | Operators run ad-hoc evaluations. | No SLA; warns if warm path > target | +| **CI** | `stella policy simulate` runs in pipelines referencing golden SBOMs. | Must complete under 10 min to avoid pipeline timeout | + +The orchestrator enforces max concurrency per tenant (`maxActiveRuns`), queue depth alarms, and fairness (round-robin per policy). + +--- + +## 7 · Monitoring & Alerts + +- **Metrics:** `policy_run_seconds`, `policy_run_queue_depth`, `policy_run_failures_total`, `policy_run_incremental_backlog`, `policy_rules_fired_total`. +- **Dashboards:** Highlight pending approvals, incremental backlog age, top failing policies, VEX override ratios (tie-in with `/docs/observability/policy.md` once published). +- **Alerts:** + - Incremental backlog > 3 cycles. + - Determinism hash mismatch. + - Failure rate > 5 % over rolling hour. + - Run duration > SLA (full > 30 min, incremental > 5 min). + +--- + +## 8 · Failure Handling & Rollback + +- **Soft failures:** Worker retries; after final failure, orchestrator emits `policy.run.failed` with diagnostics and recommended actions (e.g., missing SBOM segment). +- **Hard failures:** Schema mismatch, determinism guard violation (`ERR_POL_004`) blocks further runs until resolved. +- **Rollback:** Operators can activate previous policy version (see [Lifecycle guide](lifecycle.md)) and schedule full run to restore prior state. + +--- + +## 9 · Offline / Sealed Mode + +- Change streams originate from offline bundle imports; orchestrator processes delta manifests. +- Runs execute with `sealed=true`, blocking any external lookups; `policy_runs.inputs.env.sealed` set for auditing. +- Explain traces annotate cached data usage to prompt bundle refresh. +- Offline Kit exports include latest `policy_runs` snapshot and determinism hashes for evidence lockers. + +--- + +## 10 · Compliance Checklist + +- [ ] **Run schemas validated:** `PolicyRunRequest` / `PolicyRunStatus` DTOs from Scheduler Models (`SCHED-MODELS-20-001`) serialise deterministically; schema samples up to date. +- [ ] **Cursor integrity:** Incremental runs persist advisory & VEX cursors; replay verifies identical input digests. +- [ ] **Queue fairness configured:** Tenant-level concurrency limits and lease timeouts applied; no starvation of lower-volume policies. +- [ ] **Determinism guard active:** CI replay job (`DEVOPS-POLICY-20-003`) green; determinism hash recorded on each run. +- [ ] **Observability wired:** Metrics exported, alerts configured, and run events flowing to Notifier/Timeline. +- [ ] **Offline tested:** `stella policy run --sealed` executed in air-gapped environment; explain traces flag cached evidence usage. +- [ ] **Recovery plan rehearsed:** Failure and rollback drill documented; incident checklist aligned with Lifecycle guide. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* diff --git a/docs/runtime/SCANNER_RUNTIME_READINESS.md b/docs/runtime/SCANNER_RUNTIME_READINESS.md index cf8c9c86..56c37c4c 100644 --- a/docs/runtime/SCANNER_RUNTIME_READINESS.md +++ b/docs/runtime/SCANNER_RUNTIME_READINESS.md @@ -34,7 +34,7 @@ This runbook confirms that Scanner.WebService now surfaces the metadata Runtime ``` (Use `npm install --no-save ajv ajv-cli ajv-formats` once per clone.) -> Snapshot fixtures: see `docs/events/samples/scanner.report.ready@1.sample.json` for a canonical event that already carries `quietedFindingCount`. +> Snapshot fixtures: see `docs/events/samples/scanner.event.report.ready@1.sample.json` for a canonical orchestrator event that already carries `quietedFindingCount`. --- diff --git a/docs/schemas/policy-diff-summary.schema.json b/docs/schemas/policy-diff-summary.schema.json new file mode 100644 index 00000000..64b1a4d0 --- /dev/null +++ b/docs/schemas/policy-diff-summary.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "PolicyDiffSummary", + "type": "object", + "additionalProperties": false, + "properties": { + "SchemaVersion": { + "type": "string" + }, + "Added": { + "type": "integer", + "format": "int32" + }, + "Removed": { + "type": "integer", + "format": "int32" + }, + "Unchanged": { + "type": "integer", + "format": "int32" + }, + "BySeverity": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/PolicyDiffSeverityDelta" + } + }, + "RuleHits": { + "type": "array", + "items": { + "$ref": "#/definitions/PolicyDiffRuleDelta" + } + } + }, + "definitions": { + "PolicyDiffSeverityDelta": { + "type": "object", + "additionalProperties": false, + "properties": { + "Up": { + "type": "integer", + "format": "int32" + }, + "Down": { + "type": "integer", + "format": "int32" + } + } + }, + "PolicyDiffRuleDelta": { + "type": "object", + "additionalProperties": false, + "properties": { + "RuleId": { + "type": "string" + }, + "RuleName": { + "type": "string" + }, + "Up": { + "type": "integer", + "format": "int32" + }, + "Down": { + "type": "integer", + "format": "int32" + } + } + } + } +} diff --git a/docs/schemas/policy-explain-trace.schema.json b/docs/schemas/policy-explain-trace.schema.json new file mode 100644 index 00000000..8f928212 --- /dev/null +++ b/docs/schemas/policy-explain-trace.schema.json @@ -0,0 +1,258 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "PolicyExplainTrace", + "type": "object", + "additionalProperties": false, + "properties": { + "SchemaVersion": { + "type": "string" + }, + "FindingId": { + "type": "string" + }, + "PolicyId": { + "type": "string" + }, + "PolicyVersion": { + "type": "integer", + "format": "int32" + }, + "TenantId": { + "type": "string" + }, + "RunId": { + "type": "string" + }, + "EvaluatedAt": { + "type": "string", + "format": "date-time" + }, + "Verdict": { + "$ref": "#/definitions/PolicyExplainVerdict" + }, + "RuleChain": { + "type": "array", + "items": { + "$ref": "#/definitions/PolicyExplainRule" + } + }, + "Evidence": { + "type": "array", + "items": { + "$ref": "#/definitions/PolicyExplainEvidence" + } + }, + "VexImpacts": { + "type": "array", + "items": { + "$ref": "#/definitions/PolicyExplainVexImpact" + } + }, + "History": { + "type": "array", + "items": { + "$ref": "#/definitions/PolicyExplainHistoryEvent" + } + }, + "Metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "definitions": { + "PolicyExplainVerdict": { + "type": "object", + "additionalProperties": false, + "properties": { + "Status": { + "$ref": "#/definitions/PolicyVerdictStatus" + }, + "Severity": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/definitions/SeverityRank" + } + ] + }, + "Quiet": { + "type": "boolean" + }, + "Score": { + "type": [ + "null", + "number" + ], + "format": "double" + }, + "Rationale": { + "type": [ + "null", + "string" + ] + } + } + }, + "PolicyVerdictStatus": { + "type": "integer", + "description": "", + "x-enumNames": [ + "Passed", + "Warned", + "Blocked", + "Quieted", + "Ignored" + ], + "enum": [ + 0, + 1, + 2, + 3, + 4 + ] + }, + "SeverityRank": { + "type": "integer", + "description": "", + "x-enumNames": [ + "None", + "Info", + "Low", + "Medium", + "High", + "Critical", + "Unknown" + ], + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6 + ] + }, + "PolicyExplainRule": { + "type": "object", + "additionalProperties": false, + "properties": { + "RuleId": { + "type": "string" + }, + "RuleName": { + "type": "string" + }, + "Action": { + "type": "string" + }, + "Decision": { + "type": "string" + }, + "Score": { + "type": "number", + "format": "double" + }, + "Condition": { + "type": [ + "null", + "string" + ] + } + } + }, + "PolicyExplainEvidence": { + "type": "object", + "additionalProperties": false, + "properties": { + "Type": { + "type": "string" + }, + "Reference": { + "type": "string" + }, + "Source": { + "type": "string" + }, + "Status": { + "type": "string" + }, + "Weight": { + "type": "number", + "format": "double" + }, + "Justification": { + "type": [ + "null", + "string" + ] + }, + "Metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "PolicyExplainVexImpact": { + "type": "object", + "additionalProperties": false, + "properties": { + "StatementId": { + "type": "string" + }, + "Provider": { + "type": "string" + }, + "Status": { + "type": "string" + }, + "Accepted": { + "type": "boolean" + }, + "Justification": { + "type": [ + "null", + "string" + ] + }, + "Confidence": { + "type": [ + "null", + "string" + ] + } + } + }, + "PolicyExplainHistoryEvent": { + "type": "object", + "additionalProperties": false, + "properties": { + "Status": { + "type": "string" + }, + "OccurredAt": { + "type": "string", + "format": "date-time" + }, + "Actor": { + "type": [ + "null", + "string" + ] + }, + "Note": { + "type": [ + "null", + "string" + ] + } + } + } + } +} diff --git a/docs/schemas/policy-run-request.schema.json b/docs/schemas/policy-run-request.schema.json new file mode 100644 index 00000000..7883ea10 --- /dev/null +++ b/docs/schemas/policy-run-request.schema.json @@ -0,0 +1,130 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "PolicyRunRequest", + "type": "object", + "additionalProperties": false, + "properties": { + "SchemaVersion": { + "type": "string" + }, + "TenantId": { + "type": "string" + }, + "PolicyId": { + "type": "string" + }, + "PolicyVersion": { + "type": [ + "integer", + "null" + ], + "format": "int32" + }, + "Mode": { + "$ref": "#/definitions/PolicyRunMode" + }, + "Priority": { + "$ref": "#/definitions/PolicyRunPriority" + }, + "RunId": { + "type": [ + "null", + "string" + ] + }, + "QueuedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "RequestedBy": { + "type": [ + "null", + "string" + ] + }, + "CorrelationId": { + "type": [ + "null", + "string" + ] + }, + "Metadata": { + "type": [ + "null", + "object" + ], + "additionalProperties": { + "type": "string" + } + }, + "Inputs": { + "$ref": "#/definitions/PolicyRunInputs" + } + }, + "definitions": { + "PolicyRunMode": { + "type": "integer", + "description": "", + "x-enumNames": [ + "Full", + "Incremental", + "Simulate" + ], + "enum": [ + 0, + 1, + 2 + ] + }, + "PolicyRunPriority": { + "type": "integer", + "description": "", + "x-enumNames": [ + "Normal", + "High", + "Emergency" + ], + "enum": [ + 0, + 1, + 2 + ] + }, + "PolicyRunInputs": { + "type": "object", + "additionalProperties": false, + "properties": { + "SbomSet": { + "type": "array", + "items": { + "type": "string" + } + }, + "AdvisoryCursor": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "VexCursor": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "Environment": { + "type": "object", + "additionalProperties": {} + }, + "CaptureExplain": { + "type": "boolean" + } + } + } + } +} diff --git a/docs/schemas/policy-run-status.schema.json b/docs/schemas/policy-run-status.schema.json new file mode 100644 index 00000000..81f268a2 --- /dev/null +++ b/docs/schemas/policy-run-status.schema.json @@ -0,0 +1,217 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "PolicyRunStatus", + "type": "object", + "additionalProperties": false, + "properties": { + "SchemaVersion": { + "type": "string" + }, + "RunId": { + "type": "string" + }, + "TenantId": { + "type": "string" + }, + "PolicyId": { + "type": "string" + }, + "PolicyVersion": { + "type": "integer", + "format": "int32" + }, + "Mode": { + "$ref": "#/definitions/PolicyRunMode" + }, + "Status": { + "$ref": "#/definitions/PolicyRunExecutionStatus" + }, + "Priority": { + "$ref": "#/definitions/PolicyRunPriority" + }, + "QueuedAt": { + "type": "string", + "format": "date-time" + }, + "StartedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "FinishedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "DeterminismHash": { + "type": [ + "null", + "string" + ] + }, + "ErrorCode": { + "type": [ + "null", + "string" + ] + }, + "Error": { + "type": [ + "null", + "string" + ] + }, + "Attempts": { + "type": "integer", + "format": "int32" + }, + "TraceId": { + "type": [ + "null", + "string" + ] + }, + "ExplainUri": { + "type": [ + "null", + "string" + ] + }, + "Metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "Stats": { + "$ref": "#/definitions/PolicyRunStats" + }, + "Inputs": { + "$ref": "#/definitions/PolicyRunInputs" + } + }, + "definitions": { + "PolicyRunMode": { + "type": "integer", + "description": "", + "x-enumNames": [ + "Full", + "Incremental", + "Simulate" + ], + "enum": [ + 0, + 1, + 2 + ] + }, + "PolicyRunExecutionStatus": { + "type": "integer", + "description": "", + "x-enumNames": [ + "Queued", + "Running", + "Succeeded", + "Failed", + "Cancelled", + "ReplayPending" + ], + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5 + ] + }, + "PolicyRunPriority": { + "type": "integer", + "description": "", + "x-enumNames": [ + "Normal", + "High", + "Emergency" + ], + "enum": [ + 0, + 1, + 2 + ] + }, + "PolicyRunStats": { + "type": "object", + "additionalProperties": false, + "properties": { + "Components": { + "type": "integer", + "format": "int32" + }, + "RulesFired": { + "type": "integer", + "format": "int32" + }, + "FindingsWritten": { + "type": "integer", + "format": "int32" + }, + "VexOverrides": { + "type": "integer", + "format": "int32" + }, + "Quieted": { + "type": "integer", + "format": "int32" + }, + "Suppressed": { + "type": "integer", + "format": "int32" + }, + "DurationSeconds": { + "type": [ + "null", + "number" + ], + "format": "double" + } + } + }, + "PolicyRunInputs": { + "type": "object", + "additionalProperties": false, + "properties": { + "SbomSet": { + "type": "array", + "items": { + "type": "string" + } + }, + "AdvisoryCursor": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "VexCursor": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "Environment": { + "type": "object", + "additionalProperties": {} + }, + "CaptureExplain": { + "type": "boolean" + } + } + } + } +} diff --git a/docs/security/authority-scopes.md b/docs/security/authority-scopes.md new file mode 100644 index 00000000..0903215d --- /dev/null +++ b/docs/security/authority-scopes.md @@ -0,0 +1,194 @@ +# Authority Scopes & Tenancy — AOC Update + +> **Audience:** Authority Core, platform security engineers, DevOps owners. +> **Scope:** Scope taxonomy, tenancy enforcement, rollout guidance for the Aggregation-Only Contract (Sprint 19). + +Authority issues short-lived tokens bound to tenants and scopes. Sprint 19 introduces new scopes to support the AOC guardrails in Concelier and Excititor. This document lists the canonical scope catalogue, describes tenancy propagation, and outlines operational safeguards. + +--- + +## 1 · Scope catalogue (post AOC) + +| Scope | Surface | Purpose | Notes | +|-------|---------|---------|-------| +| `advisory:write` | Concelier ingestion APIs | Allows append-only writes to `advisory_raw`. | Granted to Concelier WebService and trusted connectors. Requires tenant claim. | +| `advisory:verify` | Concelier `/aoc/verify`, CLI, UI dashboard | Permits guard verification and access to violation summaries. | Read-only; used by `stella aoc verify` and console dashboard. | +| `vex:write` | Excititor ingestion APIs | Append-only writes to `vex_raw`. | Mirrors `advisory:write`. | +| `vex:verify` | Excititor `/aoc/verify`, CLI | Read-only verification of VEX ingestion. | Optional for environments without VEX feeds. | +| `graph:write` | Cartographer build pipeline | Enqueue graph build/overlay jobs. | Reserved for the Cartographer service identity; requires tenant claim. | +| `graph:read` | Graph API, Scheduler overlays, UI | Read graph projections/overlays. | Requires tenant claim; granted to Cartographer, Graph API, Scheduler. | +| `graph:export` | Graph export endpoints | Stream GraphML/JSONL artefacts. | UI/gateway automation only; tenant required. | +| `graph:simulate` | Policy simulation overlays | Trigger what-if overlays on graphs. | Restricted to automation; tenant required. | +| `effective:write` | Policy Engine | Allows creation/update of `effective_finding_*` collections. | **Only** the Policy Engine service client may hold this scope. | +| `effective:read` | Console, CLI, exports | Read derived findings. | Shared across tenants with role-based restrictions. | +| `aoc:dashboard` | Console UI | Access AOC dashboard resources. | Bundles `advisory:verify`/`vex:verify` by default; keep for UI RBAC group mapping. | +| `aoc:verify` | Automation service accounts | Execute verification via API without the full dashboard role. | For CI pipelines, offline kit validators. | +| Existing scopes | (e.g., `policy:*`, `sbom:*`) | Unchanged. | Review `/docs/security/policy-governance.md` for policy-specific scopes. | + +### 1.1 Scope bundles (roles) + +- **`role/concelier-ingest`** → `advisory:write`, `advisory:verify`. +- **`role/excititor-ingest`** → `vex:write`, `vex:verify`. +- **`role/aoc-operator`** → `aoc:dashboard`, `aoc:verify`, `advisory:verify`, `vex:verify`. +- **`role/policy-engine`** → `effective:write`, `effective:read`. +- **`role/cartographer-service`** → `graph:write`, `graph:read`. +- **`role/graph-gateway`** → `graph:read`, `graph:export`, `graph:simulate`. + +Roles are declared per tenant in `authority.yaml`: + +```yaml +tenants: + - name: default + roles: + concelier-ingest: + scopes: [advisory:write, advisory:verify] + aoc-operator: + scopes: [aoc:dashboard, aoc:verify, advisory:verify, vex:verify] + policy-engine: + scopes: [effective:write, effective:read] +``` + +--- + +## 2 · Tenancy enforcement + +### 2.1 Token claims + +Tokens now include: + +- `tenant` claim (string) — required for all ingestion and verification scopes. +- `service_identity` (optional) — e.g., `policy-engine`, `cartographer`. Required when requesting `effective:write` or `graph:write`. +- `delegation_allowed` (boolean) — defaults `false`. Prevents console tokens from delegating ingest scopes. + +Authority rejects requests when: + +- `tenant` is missing while requesting `advisory:*`, `vex:*`, or `aoc:*` scopes. +- `service_identity != policy-engine` but `effective:write` is present (`ERR_AOC_006` enforcement). +- `service_identity != cartographer` but `graph:write` is present (graph pipeline enforcement). +- Tokens attempt to combine `advisory:write` with `effective:write` (separation of duties). + +### 2.2 Propagation + +- API Gateway forwards `tenant` claim as header (`X-Stella-Tenant`). Services refuse requests lacking the header. +- Concelier/Excititor stamp tenant into raw documents and structured logs. +- Policy Engine copies `tenant` from tokens into `effective_finding_*` collections. + +### 2.3 Cross-tenant scenarios + +- Platform operators with `tenant:admin` can assume other tenants via `/authority/tenant/switch` if explicitly permitted. +- CLI commands accept `--tenant ` to override environment default; Authority logs tenant switch events (`authority.tenant.switch`). +- Console tenant picker uses delegated token exchange (`/token/exchange`) to obtain scoped tenant tokens without exposing raw credentials. + +--- + +## 3 · Configuration changes + +### 3.1 Authority configuration (`authority.yaml`) + +Add new scopes and optional claims transformations: + +```yaml +security: + scopes: + - name: advisory:write + description: Concelier raw ingestion + - name: advisory:verify + description: Verify Concelier ingestion + - name: vex:write + description: Excititor raw ingestion + - name: vex:verify + description: Verify Excititor ingestion + - name: aoc:dashboard + description: Access AOC UI dashboards + - name: aoc:verify + description: Run AOC verification + - name: effective:write + description: Policy Engine materialisation + - name: effective:read + description: Read derived findings + claimTransforms: + - match: { scope: "effective:write" } + require: + serviceIdentity: policy-engine + - match: { scope: "graph:write" } + require: + serviceIdentity: cartographer +``` + +### 3.2 Client registration + +Update service clients: + +- `Concelier.WebService` → request `advisory:write`, `advisory:verify`. +- `Excititor.WebService` → request `vex:write`, `vex:verify`. +- `Policy.Engine` → request `effective:write`, `effective:read`; set `properties.serviceIdentity=policy-engine`. +- `Cartographer.Service` → request `graph:write`, `graph:read`; set `properties.serviceIdentity=cartographer`. +- `Graph API Gateway` → request `graph:read`, `graph:export`, `graph:simulate`; tenant hint required. +- `Console` → request `aoc:dashboard`, `effective:read` plus existing UI scopes. +- `CLI automation` → request `aoc:verify`, `advisory:verify`, `vex:verify` as needed. + +Client definition snippet: + +```yaml +clients: + - clientId: concelier-web + grantTypes: [client_credentials] + scopes: [advisory:write, advisory:verify] + tenants: [default] + - clientId: policy-engine + grantTypes: [client_credentials] + scopes: [effective:write, effective:read] + properties: + serviceIdentity: policy-engine + - clientId: cartographer-service + grantTypes: [client_credentials] + scopes: [graph:write, graph:read] + properties: + serviceIdentity: cartographer +``` + +--- + +## 4 · Operational safeguards + +- **Audit events:** Authority emits `authority.scope.granted` and `authority.scope.revoked` events with `scope` and `tenant`. Monitor for unexpected grants. +- **Rate limiting:** Apply stricter limits on `/token` endpoints for clients requesting `advisory:write` or `vex:write` to mitigate brute-force ingestion attempts. +- **Incident response:** Link AOC alerts to Authority audit logs to confirm whether violations come from expected identities. +- **Rotation:** Rotate ingest client secrets alongside guard deployments; add rotation steps to `ops/authority-key-rotation.md`. +- **Testing:** Integration tests must fail if tokens lacking `tenant` attempt ingestion; add coverage in Concelier/Excititor smoke suites (see `CONCELIER-CORE-AOC-19-013`). + +--- + +## 5 · Offline & air-gap notes + +- Offline Kit bundles include tenant-scoped service credentials. Ensure ingest bundles ship without `advisory:write` scopes unless strictly required. +- CLI verification in offline environments uses pre-issued `aoc:verify` tokens; document expiration and renewal processes. +- Authority replicas in air-gapped environments should restrict scope issuance to known tenants and log all `/token` interactions for later replay. + +--- + +## 6 · References + +- [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) +- [Architecture overview](../architecture/overview.md) +- [Concelier architecture](../ARCHITECTURE_CONCELIER.md) +- [Excititor architecture](../ARCHITECTURE_EXCITITOR.md) +- [Policy governance](policy-governance.md) +- [Authority key rotation playbook](../ops/authority-key-rotation.md) + +--- + +## 7 · Compliance checklist + +- [ ] Scope catalogue updated in Authority configuration templates. +- [ ] Role mappings documented for each tenant profile. +- [ ] Claim transforms enforce `serviceIdentity` for `effective:write`. +- [ ] Claim transforms enforce `serviceIdentity` for `graph:write`. +- [ ] Concelier/Excititor smoke tests cover missing tenant rejection. +- [ ] Offline kit credentials reviewed for least privilege. +- [ ] Audit/monitoring guidance validated with Observability Guild. +- [ ] Authority Core sign-off recorded (owner: @authority-core, due 2025-10-28). + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/security/policy-governance.md b/docs/security/policy-governance.md new file mode 100644 index 00000000..8789de48 --- /dev/null +++ b/docs/security/policy-governance.md @@ -0,0 +1,114 @@ +# Policy Governance & Least Privilege + +> **Audience:** Security Guild, Policy Guild, Authority Core, auditors. +> **Scope:** Scopes, RBAC, approval controls, tenancy, auditing, and compliance requirements for Policy Engine v2. + +--- + +## 1 · Governance Principles + +1. **Least privilege by scope** – API clients receive only the `policy:*` scopes required for their role; `effective:write` reserved for service identity. +2. **Immutable history** – All policy changes, approvals, runs, and suppressions produce audit artefacts retrievable offline. +3. **Separation of duties** – Authors cannot approve their own submissions; approvers require distinct scope and should not have deployment rights. +4. **Deterministic verification** – Simulations, determinism checks, and incident replay bundles provide reproducible evidence for auditors. +5. **Tenant isolation** – Policies, runs, and findings scoped to tenants; cross-tenant access requires explicit admin scopes and is logged. +6. **Offline parity** – Air-gapped sites follow the same governance workflow with sealed-mode safeguards and signed bundles. + +--- + +## 2 · Authority Scopes & Role Mapping + +| Scope | Description | Recommended role | +|-------|-------------|------------------| +| `policy:read` | View policies, revisions, runs, findings. | Readers, auditors. | +| `policy:write` | Create/edit drafts, run lint/compile. | Authors (SecOps engineers). | +| `policy:submit` | Move draft → submitted, attach simulations. | Authors with submission rights. | +| `policy:review` | Comment/approve/request changes (non-final). | Reviewers (peer security, product). | +| `policy:approve` | Final approval; can archive. | Approval board/security lead. | +| `policy:activate` | Promote approved version, schedule activation. | Runtime operators / release managers. | +| `policy:run` | Trigger runs, inspect live status. | Operators, automation bots. | +| `policy:runs` | Read run history, replay bundles. | Operators, auditors. | +| `policy:archive` | Retire versions, perform rollbacks. | Approvers, operators. | +| `policy:simulate` | Execute simulations via API/CLI. | Authors, reviewers, CI. | +| `policy:operate` | Activate incident mode, toggle sampling. | SRE/on-call. | +| `findings:read` | View effective findings/explain. | Analysts, auditors, CLI. | +| `effective:write` | **Service only** – materialise findings. | Policy Engine service principal. | + +> Map organisation roles to scopes via Authority issuer config (`authority.tenants[].roles`). Document assignments in tenant onboarding checklist. + +> **Authority configuration tip:** the Policy Engine service client must include `properties.serviceIdentity: policy-engine` and a tenant hint in `authority.yaml`. Authority rejects `effective:write` tokens that lack this marker. See [Authority scopes](authority-scopes.md) for the full scope catalogue. + +--- + +## 3 · Workflow Controls + +- **Submit gate:** CLI/UI require fresh lint + simulation artefacts (<24 h). Submissions store reviewer list and diff attachments. +- **Review quorum:** Authority policy enforces minimum reviewers (e.g., 2) and optional separation between functional/security domains. +- **Approval guard:** Approvers must acknowledge simulation + determinism check completion. CLI enforces `--note` and `--attach` fields. +- **Activation guard:** Policy Engine refuses activation when latest full run status ≠ success or incremental backlog aged > SLA. +- **Rollback policy:** Rollbacks require incident reference and produce `policy.rollback` audit events. + +--- + +## 4 · Tenancy & Data Access + +- Policies stored per tenant; `tenant-global` used for shared baselines. +- API filters all requests by `X-Stella-Tenant` (default from token). Cross-tenant requests require `policy:tenant-admin`. +- Effective findings collections include `tenant` field and unique indexes preventing cross-tenant writes. +- CLI/Console display tenant context prominently; switching tenant triggers warnings when active policy differs. +- Offline bundles encode tenant metadata; import commands validate compatibility before applying. + +--- + +## 5 · Audit & Evidence + +- **Collections:** `policies`, `policy_reviews`, `policy_history`, `policy_runs`, `policy_run_events`, `effective_finding_*_history`. +- **Events:** `policy.submitted`, `policy.review.requested`, `policy.approved`, `policy.activated`, `policy.archived`, `policy.run.*`, `policy.incident.*`. +- **Explain traces:** Stored for critical findings (sampled); available via CLI/UI for auditors (requires `findings:read`). +- **Offline evidence:** `stella policy bundle export` produces DSSE-signed packages containing DSL, IR digest, simulations, approval notes, run summaries, trace metadata. +- **Retention:** Default 365 days for run history, extendable per compliance requirements; incident mode extends to 30 days minimum. + +--- + +## 6 · Secrets & Configuration Hygiene + +- Policy Engine configuration loaded from environment/secret stores; no secrets in repo. +- CLI profiles should store tokens encrypted (`stella profile set --secret`). +- UI/CLI logs redact tokens, reviewer emails, and attachments. +- Rotating tokens/keys: Authority exposes `policy scopes` in discovery docs; follow `/docs/security/authority-scopes.md` for rotation. +- Use `policy:operate` to disable self-service simulation temporarily during incident response if needed. + +--- + +## 7 · Incident Response + +- Trigger incident mode for determinism violations, backlog surges, or suspected policy abuse. +- Capture replay bundles and run `stella policy run replay` for affected runs. +- Coordinate with Observability dashboards (see `/docs/observability/policy.md`) to monitor queue depth, failures. +- After resolution, document remediation in Lifecycle guide (§8) and attach to approval history. + +--- + +## 8 · Offline / Air-Gapped Governance + +- Same scopes apply; tokens issued by local Authority. +- Approvers must use offline UI/CLI to sign submissions; attachments stored locally. +- Bundle import/export must be signed (DSSE + cosign). CLI warns if signatures missing. +- Sealed-mode banner reminds operators to refresh bundles when staleness thresholds exceeded. +- Offline audits rely on evidence bundles and local `policy_runs` snapshot. + +--- + +## 9 · Compliance Checklist + +- [ ] **Scope mapping reviewed:** Authority issuer config updated; RBAC matrix stored with change request. +- [ ] **Separation enforced:** Automated checks block self-approval; review quorum satisfied. +- [ ] **Activation guard documented:** Operators trained on run health checks before promoting. +- [ ] **Audit exports tested:** Evidence bundles verified (hash/signature) and stored per compliance policy. +- [ ] **Incident drills rehearsed:** Replay/rollback procedures executed and logged. +- [ ] **Offline parity confirmed:** Air-gapped site executes submit/approve flow with sealed-mode guidance. +- [ ] **Documentation cross-links:** References to lifecycle, runs, observability, CLI, and API docs validated. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* diff --git a/docs/ui/admin.md b/docs/ui/admin.md new file mode 100644 index 00000000..c95d6078 --- /dev/null +++ b/docs/ui/admin.md @@ -0,0 +1,174 @@ +# StellaOps Console - Admin Workspace + +> **Audience:** Authority Guild, Console admins, support engineers, tenant operators. +> **Scope:** Tenant management, role mapping, token lifecycle, integrations, fresh-auth prompts, security guardrails, offline behaviour, and compliance checklist for Sprint 23. + +The Admin workspace centralises Authority-facing controls: tenants, roles, API clients, tokens, and integrations. It surfaces RBAC mappings, token issuance logs, and bootstrap flows with the same offline-first guarantees as the rest of the console. + +--- + +## 1. Access and prerequisites + +- **Route:** `/console/admin` with sub-routes for tenants, users, roles, tokens, integrations, audit, and bootstrap. +- **Scopes:** + - `ui.admin` (base access) + - `authority:tenants.read` / `authority:tenants.write` + - `authority:roles.read` / `authority:roles.write` + - `authority:tokens.read` / `authority:tokens.revoke` + - `authority:clients.read` / `authority:clients.write` + - `authority:audit.read` (view audit trails) +- **Fresh-auth:** Sensitive actions (token revoke, bootstrap key issue, signing key rotation) require fresh-auth challenge. +- **Dependencies:** Authority service (`/internal/*` APIs), revocation export, JWKS, licensing posture endpoint, integration config store. + +--- + +## 2. Layout overview + +``` ++--------------------------------------------------------------------+ +| Header: Tenant picker - environment badge - security banner | ++--------------------------------------------------------------------+ +| Tabs: Tenants | Roles & Scopes | Users & Tokens | Integrations | Audit | ++--------------------------------------------------------------------+ +| Sidebar: Quick actions (Invite user, Create client, Export revocations) +| Main panel varies per tab | ++--------------------------------------------------------------------+ +``` + +The header includes offline status indicator and link to Authority health page. + +--- + +## 3. Tenants tab + +| Field | Description | +|-------|-------------| +| **Tenant ID** | Lowercase slug used in tokens and client registrations. | +| **Display name** | Human-friendly name. | +| **Status** | `active`, `suspended`, `pending`. Suspended tenants block token issuance. | +| **Isolation mode** | `dedicated`, `shared`, or `sandbox`. Drives RBAC defaults. | +| **Default roles** | Roles automatically assigned to new users within the tenant. | +| **Offline snapshots** | Latest snapshot timestamp, checksum, operator. | + +Actions: + +- `Create tenant` (requires `authority:tenants.write`). Form captures display name, slug, isolation mode, default roles, bootstrap contact, optional plan metadata. +- `Suspend/Resume` toggles token issuance and surfaces audit entry. +- `Export tenant bundle` downloads tenant-specific revocation + JWKS package for air-gap distribution. +- CLI parity: `stella auth tenant create --tenant `, `stella auth tenant suspend --tenant `. + +--- + +## 4. Roles & scopes tab + +- Table lists roles with mapped scopes and audiences. +- Inline editor supports adding/removing scopes (with validation). +- Scope categories: UI, Scanner, Concelier, Excititor, Policy, Attestor, Notifier, Scheduler, Offline kit. +- Visual diff shows impact of changes on linked clients/users before committing. +- "Effective permissions" view summarises what each role grants per service. +- CLI parity: `stella auth role update --role ui.admin --add-scope authority:tokens.revoke`. + +--- + +## 5. Users & tokens tab + +Sections: + +1. **User list** - identity, tenant, roles, last login, MFA status. Actions include reset password (if plugin supports), enforce fresh-auth, disable user. +2. **Token inventory** - lists active tokens (access/refresh/device). Columns: token ID, type, subject, audience, issued at, expires, status. Toggle to show revoked tokens. +3. **Token details** drawer shows claims, sender constraint (`cnf`), issuance metadata, revocation history. +4. **Revoke token** action requires fresh-auth and prompts for reason (incident, user request, compromise). +5. **Bulk revoke** (per tenant or role) triggers Authority revocation export to ensure downstream services purge caches. + +Audit entries appear for every user/token change. CLI parity: `stella auth token revoke --token `. + +--- + +## 6. Integrations tab + +- **Authority clients** list (service accounts) with grant types, allowed scopes, DPoP/mTLS settings, tenant hints, and rotation status. +- **Bootstrap bundles** - downloadable templates for new clients/users; includes configuration YAML and CLI instructions. +- **External IdP connectors** (optional) - displays status for SAML/OIDC plugins; includes metadata upload field and test login result. +- **Licensing posture** - read-only panel summarising plan tier, entitlement expiry, and contact info (pulled from licensing service). +- **Notifications** - optional webhook configuration for token events (on revoke, on failure). +- CLI parity: `stella auth client create --client concelier --grant client_credentials --tenant prod`. + +--- + +## 7. Audit tab + +- Timeline view of administrative events (user changes, role updates, token revocations, bootstrap actions, key rotations). +- Filters: event type, actor, tenant, scope, correlation ID. +- Export button downloads CSV/JSON for SOC ingestion. +- "Open in logs" copies search query pre-populated with correlation IDs. +- CLI parity: `stella auth audit export --from 2025-10-20`. + +--- + +## 8. Fresh-auth prompts + +- High-risk actions (revoke all tokens, rotate signing key, create privileged client) trigger modal requiring credential re-entry or hardware key touch. +- Fresh-auth window is 5 minutes; countdown displayed. +- UI surface indicates when current session is outside fresh-auth window; sensitive buttons disabled until re-auth. +- Audit log records fresh-auth events (`authority.fresh_auth.start`, `authority.fresh_auth.success`). +- CLI parity: `stella auth fresh-auth` obtains short-lived token for scriptable flows. + +--- + +## 9. Security guardrails + +- DPoP enforcement reminders for UI clients; console warns if any client lacks sender constraint. +- mTLS enforcement summary for high-value audiences (Signer/Attestor). +- Token policy checklists (access token TTL, refresh token policy) with alerts when deviating from defaults. +- Revocation bundle export status (timestamp, digest, operator). +- Key rotation panel showing current `kid`, last rotation, next scheduled rotation, and manual trigger button (ties into Authority rotate API). +- CLI parity: `stella auth signing rotate` for script automation. + +--- + +## 10. Offline and air-gap behaviour + +- Offline banner indicates snapshot version; disables direct remote calls. +- Tenant/role edits queue change manifests; UI instructs users to apply via CLI (`stella auth apply --bundle `). +- Token inventory shows snapshot state; revoke buttons generate scripts for offline Authority host. +- Integrations tab offers manual download/upload for client definitions and IdP metadata. +- Audit exports default to local storage with checksum output for transfer. + +--- + +## 11. Screenshot coordination + +- Placeholders: + - `![Admin tenants placeholder](../assets/ui/admin/tenants-placeholder.png)` + - `![Admin roles placeholder](../assets/ui/admin/roles-placeholder.png)` + - `![Admin tokens placeholder](../assets/ui/admin/tokens-placeholder.png)` +- Capture real screenshots with Authority Guild once Sprint 23 UI is final (tracked in `#console-screenshots`, 2025-10-26 entry). Provide both light and dark theme variants. + +--- + +## 12. References + +- `/docs/ARCHITECTURE_AUTHORITY.md` - Authority architecture. +- `/docs/11_AUTHORITY.md` - Authority service overview. +- `/docs/security/authority-scopes.md` - scope definitions. +- `/docs/ui/policies.md` - policy approvals requiring fresh-auth. +- `/docs/ui/console-overview.md` - navigation shell. +- `/docs/cli/authentication.md` (pending) and `/docs/cli/policy.md` for CLI flows. +- `/docs/ops/scheduler-runbook.md` for integration with scheduler token rotation. + +--- + +## 13. Compliance checklist + +- [ ] Tenants, roles/scopes, and token management documented with actions and CLI parity. +- [ ] Integrations and audit views covered. +- [ ] Fresh-auth prompts and guardrails described. +- [ ] Security controls (DPoP, mTLS, key rotation, revocations) captured. +- [ ] Offline behaviour explained with script guidance. +- [ ] Screenshot placeholders and coordination noted. +- [ ] References validated. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* + diff --git a/docs/ui/advisories-and-vex.md b/docs/ui/advisories-and-vex.md new file mode 100644 index 00000000..722a6940 --- /dev/null +++ b/docs/ui/advisories-and-vex.md @@ -0,0 +1,199 @@ +# StellaOps Console - Advisories and VEX + +> **Audience:** Console UX team, Concelier and Excititor guilds, support and compliance engineers. +> **Scope:** Advisory aggregation UX, VEX consensus display, conflict indicators, raw document viewer, provenance banners, CLI parity, and Aggregation-Only Contract (AOC) guardrails for Sprint 23. + +The Advisories and VEX surfaces expose Concelier and Excititor outputs without mutating the underlying data. Operators can review upstream statements, check consensus summaries, inspect conflicts, and hand off evidence to downstream tooling while staying within the Aggregation-Only Contract. + +--- + +## 1. Access and prerequisites + +- **Routes:** + - `/console/advisories` (advisory list and detail) + - `/console/vex` (VEX consensus and raw claim explorer) +- **Scopes:** `advisory.read` and `vex.read` (base access), `advisory.verify` / `vex.verify` for verification actions, `downloads.read` for evidence exports. +- **Feature flags:** `advisoryExplorer.enabled`, `vexExplorer.enabled`, `aggregation.conflictIndicators`. +- **Dependencies:** Concelier WebService (aggregation API + delta metrics), Excititor WebService (consensus API + conflict feeds), Policy Engine explain hints (optional link-outs), Authority tenant enforcement. +- **Offline behaviour:** Uses Offline Kit snapshots when gateway is in sealed mode; verify buttons queue until connectivity resumes. + +--- + +## 2. Layout overview + +``` ++---------------------------------------------------------------------+ +| Header: Tenant badge - global filters - status ticker - actions | ++---------------------------------------------------------------------+ +| Left rail: Saved views - provider filters - verification queue | ++---------------------------------------------------------------------+ +| Main split pane | +| - Advisories tab (grid + detail drawer) | +| - VEX tab (consensus table + claim drawer) | +| Tabs remember last active view per tenant. | ++---------------------------------------------------------------------+ +``` + +The header reuses console-wide context chips (`Tenant`, `Severity`, `Source`, `Time`) and the status ticker that streams Concelier and Excititor deltas. + +--- + +## 3. Advisory aggregation view + +| Element | Description | +|---------|-------------| +| **Grid columns** | Vulnerability key (CVE/GHSA/vendor), Title, Source set, Last merged, Severity badge, KEV flag, Affected product count, Merge hash. | +| **Source chips** | Show contributing providers (NVD, Red Hat, Debian, vendor PSIRT). Hover reveals precedence order and timestamps. | +| **Severity** | Displays the highest severity declared by any source; tooltip lists per-source severities and vectors. | +| **KEV / Exploit status** | Badge highlights known exploited status from Concelier enrichment; links to KEV reference. | +| **Merge hash** | Deterministic hash from Concelier `merge_event`. Clicking copies hash and opens provenance banner. | +| **Filters** | Vulnerability identifier search, provider multi-select, severity picker, KEV toggle, affected product range slider, time window. | +| **List actions** | `Open detail`, `Copy CLI` (`stella advisory show ...`), `Compare sources`, `Queue verify`. | + +The grid virtualises up to 15,000 advisories per tenant. Beyond that, the UI engages server-side pagination with cursor hints supplied by Concelier. + +--- + +## 4. Advisory detail drawer + +Sections within the drawer: + +1. **Summary cards** (title, published/modified timestamps, advisory merge hash, total sources, exploited flag). +2. **Sources timeline** listing each contributing document with signature status, fetched timestamps, precedence rank, and quick links to raw view. +3. **Affected products** table (product key, introduced/fixed, range semantics, distro qualifiers, notes). Column toggles allow switching between SemVer and distro notation. +4. **Conflict indicators** show when sources disagree on fixed versions, severity, or affected sets. Each conflict row links to an explainer panel that describes the winning value, losing sources, and precedence rule. +5. **References** collapsible list (patches, advisories, exploits). +6. **Raw JSON** viewer (read-only) using canonical Concelier payload. Users can copy JSON or download via `GET /console/advisories/raw/{id}`. +7. **CLI parity** card with commands: + - `stella advisory show --tenant --vuln ` + - `stella advisory sources --tenant --vuln ` + - `stella advisory export --tenant --vuln --format cdx-json` + +Provenance banner at the top indicates whether all sources are signed, partially signed, or unsigned, referencing AOC guardrails. Unsigned sources trigger a warning and link to the verification checklist. + +--- + +## 5. VEX explorer + +| Feature | Description | +|---------|-------------| +| **Consensus table** | Rows keyed by `(vulnId, productKey)` with rollup status (affected, not affected, fixed, under investigation), confidence score, provider count, and last evaluation timestamp. | +| **Status badges** | Colour-coded (red affected, green not affected, blue fixed, amber under investigation). Tooltips show justification and policy revision used. | +| **Provider breakdown** | Hover or expand to see source list with accepted/ignored flag, status, justification code, signature state, weight. | +| **Filters** | Product search (PURL), status filter, provider filter, justification codes, confidence threshold slider. | +| **Saved views** | Prebuilt presets: `Vendor consensus`, `Distro overrides`, `Conflicts`, `Pending investigation`. | + +--- + +## 6. VEX detail drawer + +Tabs within the drawer: + +- **Consensus summary**: Restates rollup status, policy revision, confidence benchmarks, and referencing runs. +- **Claims list**: Every raw claim from Excititor with provenance, signature result, justification, supersedes chain, evidence snippets. Claims are grouped by provider tier (vendor, distro, ecosystem, CERT). +- **Conflict explainers**: For conflicting claims, shows why a claim was ignored (weight, stale timestamp, failing justification gate). Includes inline diff between competing claims. +- **Events**: Timeline of claim arrivals and consensus evaluations with correlation IDs, accessible for debugging. +- **Raw JSON**: Canonical `VexClaim` or `VexConsensus` payloads with copy/download. CLI parity callouts: + - `stella vex consensus show --tenant --vuln --product ` + - `stella vex claims show --tenant --vuln --provider ` + +--- + +## 7. Raw viewers and provenance + +- Raw viewers display canonical payloads with syntax highlighting and copy-as-JSON support. +- Provenance banner presents: source URI, document digest, signature status, fetch timestamps, collector version. +- Users can open raw documents in a modal that includes: + - `sha256` digest with copy button + - Signature verification summary (passing keys, missing signatures, errors) + - `Download DSSE bundle` button when the document is attested + - `Open in logs` link that copies search query (`correlationId=...`) for log aggregation tools. + +All raw views are read-only to maintain Aggregation-Only guarantees. + +--- + +## 8. Conflict indicators and aggregation-not-merge UX + +- Concelier retains every source; the UI surfaces conflicts rather than merging them. +- Conflict badges appear in grids and detail views when sources disagree on affected ranges, fixed versions, severity, or exploit flags. +- Clicking a badge opens the conflict explainer panel (powered by Concelier merge metadata) that lists winning/losing sources, ranks, and reasoning (e.g., "Vendor PSIRT overrides ecosystem advisory"). +- Excititor conflicts highlight discarded claims with reasons (stale, failing justification, low weight). Operators can override weights downstream via Policy Engine if needed. +- UI copy explicitly reminds users that policy decisions happen elsewhere; these views show aggregated facts only. + +--- + +## 9. Verification workflows + +- **Run verify** buttons call Concelier or Excititor verification endpoints (`POST /console/advisories/verify`, `POST /console/vex/verify`) scoped by tenant and source filters. +- Verification results appear as banners summarising documents checked, signatures verified, and guard violations. +- Failed verifications show actionable error IDs (`ERR_AOC_00x`), matching CLI output. +- Verification history accessible via the status ticker dropdown; entries include operator, scope, and correlation IDs. + +--- + +## 10. Exports and automation + +- Advisory tab exposes export actions: `Download normalized advisory`, `Download affected products CSV`, `Download source bundle` (raw documents packaged with manifest). +- VEX tab supports exports for consensus snapshots, raw claims, and provider deltas. +- Export manifests include merge hash or consensus digest, tenant ID, timestamp, and signature state. +- CLI parity snippets accompany each export (e.g., `stella advisory export`, `stella vex export`). +- Automation: copy buttons for webhook subscription (`/downloads/hooks/subscribe`) and ORAS push commands when using remote registries. + +--- + +## 11. Observability and SSE updates + +- Status ticker shows ingest lag (`advisory_delta_minutes`, `vex_delta_minutes`), last merge event hash, and verification queue depth. +- Advisory and VEX grids refresh via SSE channels; updates animate row badges (new source, conflict resolved). +- Metrics surfaced in drawers: ingestion age, signature pass rate, consensus evaluation duration. +- Errors display correlation IDs linking to Concelier/Excititor logs. + +--- + +## 12. Offline and air-gap behaviour + +- When offline, list views display snapshot badge, staleness timer, and disable real-time verification. +- Raw downloads reference local snapshot directories and include checksum instructions. +- Exports queue locally; UI offers `Copy to removable media` instructions. +- CLI parity switches to offline commands (`--offline`, `--snapshot`). +- Tenant picker hides tenants not present in the snapshot to avoid partial data views. + +--- + +## 13. Screenshot coordination + +- Placeholders: + - `![Advisory grid placeholder](../assets/ui/advisories/grid-placeholder.png)` + - `![VEX consensus placeholder](../assets/ui/advisories/vex-placeholder.png)` +- Coordinate with Console Guild to capture updated screenshots (dark and light themes) once Sprint 23 build candidate is tagged. Tracking in Slack channel `#console-screenshots` (entry 2025-10-26). + +--- + +## 14. References + +- `/docs/ui/console-overview.md` - shell, filters, tenant model. +- `/docs/ui/navigation.md` - command palette, deep-link schema. +- `/docs/ingestion/aggregation-only-contract.md` - AOC guardrails. +- `/docs/architecture/CONCELIER.md` - merge rules, provenance. +- `/docs/architecture/EXCITITOR.md` - VEX consensus model. +- `/docs/security/console-security.md` - scopes, DPoP, CSP. +- `/docs/cli-vs-ui-parity.md` - CLI equivalence matrix. + +--- + +## 15. Compliance checklist + +- [ ] Advisory grid columns, filters, and merge hash behaviour documented. +- [ ] VEX consensus view covers status badges, provider breakdown, and filters. +- [ ] Raw viewer and provenance banners explained with AOC alignment. +- [ ] Conflict indicators and explainers tied to aggregation-not-merge rules. +- [ ] Verification workflow and CLI parity documented. +- [ ] Offline behaviour and automation paths captured. +- [ ] Screenshot placeholders and coordination notes recorded. +- [ ] References validated. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* + diff --git a/docs/ui/console-overview.md b/docs/ui/console-overview.md new file mode 100644 index 00000000..bf87706c --- /dev/null +++ b/docs/ui/console-overview.md @@ -0,0 +1,130 @@ +# StellaOps Console – Overview + +> **Audience:** Console product leads, Docs Guild writers, backend/API partners. +> **Scope:** Information architecture, tenant scoping, global filters, and Aggregation‑Only Contract (AOC) alignment for the unified StellaOps Console that lands with Sprint 23. + +The StellaOps Console is the single entry point for operators to explore SBOMs, advisories, policies, runs, and administrative surfaces. This overview explains how the console is organised, how users move between tenants, and how shared filters keep data views consistent across modules while respecting AOC boundaries. + +--- + +## 1 · Mission & Principles + +- **Deterministic navigation.** Every route is stable and deep-link friendly. URLs carry enough context (tenant, filter tokens, view modes) to let operators resume work without reapplying filters. +- **Tenant isolation first.** Any cross-tenant action requires fresh authority, and cross-tenant comparisons are made explicit so users never accidentally mix data sets. +- **Aggregation-not-merge UX.** Console surfaces advisory and VEX rollups exactly as produced by Concelier and Excititor—no client-side re-weighting or mutation. +- **Offline parity.** Every view has an offline equivalent powered by Offline Kit bundles or cached data, and exposes the staleness budget prominently. + +--- + +## 2 · Information Architecture + +### 2.1 Primary navigation + +``` +Console Root + ├─ Dashboard # KPIs, alerts, feed age, queue depth + ├─ Findings # Aggregated vulns + explanations (Policy Engine) + ├─ SBOM Explorer # Catalog, component graph, overlays + ├─ Advisories & VEX # Concelier / Excititor aggregation outputs + ├─ Runs # Scheduler runs, scan evidence, retry controls + ├─ Policies # Editor, simulations, approvals + ├─ Downloads # Signed artifacts, Offline Kit parity + ├─ Admin # Tenants, roles, tokens, integrations + └─ Help & Tours # Contextual docs, guided walkthroughs +``` + +Routes lazy-load feature shells so the UI can grow without increasing first-paint cost. Each feature owns its sub-navigation and exposes a `KeyboardShortcuts` modal describing the available accelerators. + +### 2.2 Shared surfaces + +| Surface | Purpose | Notes | +|---------|---------|-------| +| **Top bar** | Shows active tenant, environment badge (prod/non-prod), offline status pill, user menu, notifications inbox, and the command palette trigger (`⌘/Ctrl K`). | Offline status turns amber when data staleness exceeds configured thresholds. | +| **Global filter tray** | Expands from the right edge (`Shift F`). Hosts universal filters (tenant, time window, tags, severity) that apply across compatible routes. | Filter tray remembers per-tenant presets; stored in IndexedDB (non-sensitive). | +| **Context chips** | Display active global filters underneath page titles, with one-click removal (`⌫`). | Chips include the origin (e.g., `Tenant: west-prod`). | +| **Status ticker** | SSE-driven strip that surfaces Concelier/Excititor ingestion deltas, scheduler lag, and attestor queue depth. | Pulls from `/console/status` proxy (see WEB-CONSOLE-23-002). | + +--- + +## 3 · Tenant Model + +| Aspect | Detail | +|--------|--------| +| **Tenant sources** | The console obtains the tenant list and metadata from Authority `/v1/tenants` after login. Tenant descriptors include display name, slug, environment tag, and RBAC hints (role mask). | +| **Selection workflow** | First visit prompts for a default tenant. Afterwards, the tenant picker (`⌘/Ctrl T`) switches context without full reload, issuing `Authorization` refresh with the new tenant scope. | +| **Token handling** | Each tenant change generates a short-lived, DPoP-bound access token (`aud=console`, `tenant=`). Tokens live in memory; metadata persists in `sessionStorage` for reload continuity. | +| **Cross-tenant comparisons** | Side-by-side dashboards (Dashboard, Findings, SBOM Explorer) allow multi-tenant comparison only via explicit *"Add tenant"* control. Requests issue parallel API calls with separate tokens; results render in split panes labelled per tenant. | +| **Fresh-auth gated actions** | Admin and policy approvals call `Authority /fresh-auth` before executing. UI enforces a 5-minute window; afterwards, actions remain visible but disabled pending re-auth. | +| **Audit trail** | Tenant switches emit structured logs (`action=ui.tenant.switch`, `tenantId`, `subject`, `previousTenant`) and appear in Authority audit exports. | + +### 3.1 Offline operation + +In offline or sealed environments, the tenant picker only lists tenants bundled within the Offline Kit snapshot. Switching tenants prompts an "offline snapshot" banner showing the snapshot timestamp. Actions that require round-trips to Authority (fresh-auth, token rotation) show guidance to perform the step on an online bastion and import credentials later. + +--- + +## 4 · Global Filters & Context Tokens + +| Filter | Applies To | Source & Behaviour | +|--------|------------|--------------------| +| **Tenant** | All modules | Primary isolation control. Stored in URL (`?tenant=`) and via `x-tenant-id` header injected by the web proxy. Changes invalidate cached data stores. | +| **Time window** | Dashboard, Findings, Advisories & VEX, Runs | Options: `24 h`, `7 d`, `30 d`, custom ISO range. Default aligns with Compliance/Authority reporting window. Shared via query param `since=`/`until=`. | +| **Severity / Impact** | Findings, Advisories & VEX, SBOM Explorer overlays | Multi-select (Critical/High/Medium/Low/Informational, plus `Exploited` tag). Values map to Policy Engine impact buckets and Concelier KEV flags. | +| **Component tags** | SBOM Explorer, Findings | Tags drawn from SBOM metadata (`component.tags[]`). Includes search-as-you-type with scoped suggestions (package type, supplier, license). | +| **Source providers** | Advisories & VEX | Filter by provider IDs (e.g., NVD, GHSA, vendor VEX). Tied to Aggregation-Only provenance; filtering never alters base precedence. | +| **Run status** | Runs, Dashboard | States: `queued`, `running`, `completed`, `failed`, `cancelled`. Pulled from Scheduler SSE stream; default shows non-terminal states. | +| **Policy view** | Findings, Policies | Toggles between Active policy, Staged policy, and Simulation snapshots. Selecting Simulation requires prior simulation run; console links to create one if absent. | + +Filters emit deterministic tokens placed in the URL hash for copy/paste parity with CLI commands (see `/docs/cli-vs-ui-parity.md`). The console warns when a filter combination has no effect on the current view and offers to reset to defaults. + +### 4.1 Presets & Saved Views + +Users can save a set of global filters as named presets (stored per tenant). Presets show up in the command palette and the dashboard landing cards for quick access (`⌘/Ctrl 1..9`). + +--- + +## 5 · Aggregation-Only Alignment + +- **Read-only aggregation.** Pages that list advisories or VEX claims consume the canonical aggregation endpoints (`/console/advisories`, `/console/vex`). They never merge or reconcile records client-side. Instead, they highlight the source lineage and precedence as supplied by Concelier and Excititor. +- **Consistency indicators.** Each aggregated item displays source badges, precedence order, and a "last merge event hash" so operators can cross-reference Concelier logs. When a source is missing or stale, the UI surfaces a provenance banner linking to the raw document. +- **AOC guardrails.** Workflow actions (e.g., "request verify", "download evidence bundle") route through Concelier WebService guard endpoints that enforce Aggregation-Only rules. UI strings reinforce that policy decisions happen in Policy Engine, not here. +- **Audit alignment.** Any cross-navigation from aggregated data into findings or policies preserves the underlying IDs so analysts can track how aggregated data influences policy verdicts without altering the data itself. +- **CLI parity.** Inline callouts copy the equivalent `stella` CLI commands, ensuring console users can recreate the exact aggregation query offline. + +--- + +## 6 · Performance & Telemetry Anchors + +- Initial boot target: **< 2.5 s** `LargestContentfulPaint` on 4 vCPU air-gapped runner with cached assets. +- Route budget: each feature shell must keep first interaction (hydrated data + filters) under **1.5 s** once tokens resolve. +- Telemetry: console emits metrics via the `/console/telemetry` batch endpoint—`ui_route_render_seconds`, `ui_filter_apply_total`, `ui_tenant_switch_total`, `ui_offline_banner_seconds`. Logs carry correlation IDs matching backend responses for unified tracing. +- Lighthouse CI runs in the console pipeline (see `DEVOPS-CONSOLE-23-001`) and asserts budgets above; failing runs gate releases. + +--- + +## 7 · References + +- `/docs/architecture/console.md` – component-level diagrams (pending Sprint 23 task). +- `/docs/ui/navigation.md` – detailed routes, breadcrumbs, keyboard shortcuts. +- `/docs/ui/downloads.md` – downloads manifest, parity workflows, offline guidance. +- `/docs/ui/sbom-explorer.md` – SBOM-specific flows and overlays. +- `/docs/ui/advisories-and-vex.md` – aggregation UX details. +- `/docs/ui/findings.md` – explain drawer and filter matrix. +- `/docs/security/console-security.md` – OIDC, scopes, CSP, evidence handling. +- `/docs/cli-vs-ui-parity.md` – CLI equivalents and regression automation. + +--- + +## 8 · Compliance Checklist + +- [ ] Tenant picker enforces Authority-issued scopes and logs `ui.tenant.switch`. +- [ ] Global filters update URLs/query tokens for deterministic deep links. +- [ ] Aggregation views show provenance badges and merge hash indicators. +- [ ] CLI parity callouts aligned with `stella` commands for equivalent queries. +- [ ] Offline banner tested with Offline Kit snapshot import and documented staleness thresholds. +- [ ] Accessibility audit covers global filter tray, tenant picker, and keyboard shortcuts (WCAG 2.2 AA). +- [ ] Telemetry and Lighthouse budgets tracked in console CI (`DEVOPS-CONSOLE-23-001`). + +--- + +*Last updated: 2025-10-26 (Sprint 23).* diff --git a/docs/ui/console.md b/docs/ui/console.md new file mode 100644 index 00000000..b28e1849 --- /dev/null +++ b/docs/ui/console.md @@ -0,0 +1,143 @@ +# Console AOC Dashboard + +> **Audience:** Console PMs, UI engineers, Concelier/Excititor operators, SREs monitoring ingestion health. +> **Scope:** Layout, RBAC, workflow, and observability for the Aggregation-Only Contract (AOC) dashboard that ships with Sprint 19. + +The Console AOC dashboard gives operators a live view of ingestion guardrails across all configured sources. It surfaces raw Concelier/Excititor health, highlights violations raised by `AOCWriteGuard`, and lets on-call staff trigger verification without leaving the browser. Use it alongside the [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) and the [architecture overview](../architecture/overview.md) when rolling out AOC changes. + +--- + +## 1 · Access & prerequisites + +- **Route:** `/console/sources` (dashboard) with contextual drawer routes `/console/sources/:sourceKey` and `/console/sources/:sourceKey/violations/:documentId`. +- **Feature flag:** `aocDashboard.enabled` (default `true` once Concelier WebService exposes `/aoc/verify`). Toggle is tenant-scoped to support phased rollout. +- **Scopes:** + - `ui.read` (base navigation) and `advisory:verify` to view ingestion stats/violations. + - `vex:verify` to see Excititor entries and run VEX verifications. + - `advisory:write` / `vex:write` **not** required; dashboard uses read-only APIs. +- **Tenancy:** All data is filtered by the active tenant selector. Switching tenants re-fetches tiles and drill-down tables with tenant-scoped tokens. +- **Back-end contracts:** Requires Concelier/Excititor 19.x (AOC guards enabled) and Authority scopes updated per [Authority service docs](../ARCHITECTURE_AUTHORITY.md#new-aoc-scopes). + +--- + +## 2 · Layout overview + +``` +┌────────────────────────────────────────────────────────────────────────────┐ +│ Header: tenant picker • live status pill • Last verify (“2h ago”) │ +├────────────────────────────────────────────────────────────────────────────┤ +│ Tile grid (4 per row) │ +│ ┌───── Concelier sources ─────┐ ┌────── Excititor sources ────────┐ │ +│ │ Red Hat | Ubuntu | OSV ... │ │ Vendor VEX | CSAF feeds ... │ │ +├────────────────────────────────────────────────────────────────────────────┤ +│ Violations & history table │ +│ • Filters: timeframe, source, ERR_AOC code, severity (warning/block) │ +│ • Columns: timestamp, source, code, summary, supersedes link, actions │ +├────────────────────────────────────────────────────────────────────────────┤ +│ Action bar: Run Verify • Download CSV • Open Concelier raw doc • Help │ +└────────────────────────────────────────────────────────────────────────────┘ +``` + +Tiles summarise the latest ingestion runs. The table and drawers provide drill-down views, and the action bar launches verifier workflows or exports evidence for audits. + +--- + +## 3 · Source tiles + +Each tile represents a Concelier or Excititor source and contains the fields below. + +| Field | Description | Thresholds & colours | +| ------ | ----------- | -------------------- | +| **Status badge** | Aggregated health computed from the latest job. | `Healthy` (green) when last job finished < 30 min ago and `violations24h = 0`; `Warning` (amber) when age ≥ 30 min or ≤ 5 violations; `Critical` (red) on any guard rejection (`ERR_AOC_00x`) or if job age > 2 h. | +| **Last ingest** | Timestamp and relative age of last successful append to `advisory_raw`/`vex_raw`. | Clicking opens job detail drawer. | +| **Violations (24 h)** | Count of guard failures grouped by `ERR_AOC` code across the last 24 hours. | Shows pill per code (e.g., `ERR_AOC_001 × 2`). | +| **Supersedes depth** | Average length of supersedes chain for the source over the last day. | Helps spot runaway revisions. | +| **Signature pass rate** | % of documents where signature/checksum verification succeeded. | Derived from `ingestion_signature_verified_total`. | +| **Latency P95** | Write latency recorded by ingestion spans / histograms. | Mirrors `ingestion_latency_seconds{quantile=0.95}`. | + +Tile menus expose quick actions: + +- **View history** – jumps to table filtered by the selected source. +- **Open metrics** – deep links to Grafana panel seeded with `source=` for `ingestion_write_total` and `aoc_violation_total`. +- **Download raw sample** – fetches the most recent document via `GET /advisories/raw/{id}` (or VEX equivalent) for debugging. + +--- + +## 4 · Violation drill-down workflow + +1. **Select a tile** or use table filters to focus on a source, timeframe, or `ERR_AOC` code. +2. **Inspect the violation row:** summary shows offending field, guard code, and document hash. +3. **Open detail drawer:** reveals provenance (source URI, signature info), supersedes chain, and raw JSON (redacted secrets). Drawer also lists linked `effective_finding_*` entries if Policy Engine has already materialised overlays. +4. **Remediate / annotate:** operators can add notes (stored as structured annotations) or flag as *acknowledged* (for on-call rotations). Annotations sync to Concelier audit logs. +5. **Escalate:** “Create incident” button opens the standard incident template pre-filled with context (requires `ui.incidents` scope). + +The drill-down retains filter state, so back navigation returns to the scoped table without reloading the entire dashboard. + +--- + +## 5 · Verification & actions + +- **Run Verify:** calls `POST /aoc/verify` with the chosen `since` window (default 24 h). UI displays summary cards (documents checked, violations found, top codes) and stores reports for 7 days. Results include a downloadable JSON manifest mirroring CLI output. +- **Schedule verify:** schedule modal configures automated verification (daily/weekly) and optional email/Notifier hooks. +- **Export evidence:** CSV/JSON export buttons include tile metrics, verification summaries, and violation annotations—useful for audits. +- **Open in CLI:** copies `stella aoc verify --tenant --since ` for parity with automation scripts. + +All verify actions are scoped by tenant and recorded in Authority audit logs (`action=aoc.verify.ui`). + +--- + +## 6 · Metrics & observability + +The dashboard consumes the same metrics emitted by Concelier/Excititor (documented in the [AOC reference](../ingestion/aggregation-only-contract.md#9-observability-and-diagnostics)): + +- `ingestion_write_total{source,tenant,result}` – populates success/error sparklines beneath each tile. +- `aoc_violation_total{source,tenant,code}` – feeds violation pills and trend chart. +- `ingestion_signature_verified_total{source,result}` – renders signature pass-rate gauge. +- `ingestion_latency_seconds{source,quantile}` – used for latency badges and alert banners. +- `advisory_revision_count{source}` – displayed in supersedes depth tooltip. + +The page shows the correlation ID for each violation entry, matching structured logs emitted by Concelier and Excititor, enabling quick log pivoting. + +--- + +## 7 · Security & tenancy + +- Tokens are DPoP-bound; every API call includes the UI’s DPoP proof and inherits tenant scoping from Authority. +- Violations drawer hides sensitive fields (credentials, private keys) using the same redaction rules as Concelier events. +- Run Verify honours rate limits to avoid overloading ingestion services; repeated failures trigger a cool-down banner. +- The dashboard never exposes derived severity or policy status—only raw ingestion facts and guard results, preserving AOC separation of duties. + +--- + +## 8 · Offline & air-gap behaviour + +- In sealed/offline mode the dashboard switches to **“offline snapshot”** banner, reading from Offline Kit snapshots seeded via `ouk` imports. +- Verification requests queue until connectivity resumes; UI provides `Download script` to run `stella aoc verify` on a workstation and upload results later. +- Tiles display the timestamp of the last imported snapshot and flag when it exceeds the configured staleness threshold (default 48 h offline). +- CSV/JSON exports include checksums so operators can transfer evidence across air gaps securely. + +--- + +## 9 · Related references + +- [Aggregation-Only Contract reference](../ingestion/aggregation-only-contract.md) +- [Architecture overview](../architecture/overview.md) +- [Concelier architecture](../ARCHITECTURE_CONCELIER.md) +- [Excititor architecture](../ARCHITECTURE_EXCITITOR.md) +- [CLI AOC commands](../cli/cli-reference.md) + +--- + +## 10 · Compliance checklist + +- [ ] Dashboard wired to live AOC metrics (`ingestion_*`, `aoc_violation_total`). +- [ ] Verify action logs to Authority audit trail with tenant context. +- [ ] UI enforces read-only access to raw stores; no mutation endpoints invoked. +- [ ] Offline/air-gap mode documented and validated with Offline Kit snapshots. +- [ ] Violation exports include provenance and `ERR_AOC_00x` codes. +- [ ] Accessibility tested (WCAG 2.2 AA) for tiles, tables, and drawers. +- [ ] Screenshot/recording captured for Docs release notes (pending UI capture). + +--- + +*Last updated: 2025-10-26 (Sprint 19).* diff --git a/docs/ui/downloads.md b/docs/ui/downloads.md new file mode 100644 index 00000000..2cc39bdf --- /dev/null +++ b/docs/ui/downloads.md @@ -0,0 +1,212 @@ +# StellaOps Console - Downloads Manager + +> **Audience:** DevOps guild, Console engineers, enablement writers, and operators who promote releases or maintain offline mirrors. +> **Scope:** `/console/downloads` workspace covering artifact catalog, signed manifest plumbing, export status handling, CLI parity, automation hooks, and offline guidance (Sprint 23). + +The Downloads workspace centralises every artefact required to deploy or validate StellaOps in connected and air-gapped environments. It keeps Console operators aligned with release engineering by surfacing the signed downloads manifest, live export jobs, parity checks against Offline Kit bundles, and automation hooks that mirror the CLI experience. + +--- + +## 1 - Access and prerequisites + +- **Route:** `/console/downloads` (list) with detail drawer `/console/downloads/:artifactId`. +- **Scopes:** `downloads.read` (baseline) and `downloads.manage` for cancelling or expiring stale exports. Evidence bundles inherit the originating scope (`runs.read`, `findings.read`, etc.). +- **Dependencies:** Web gateway `/console/downloads` API (WEB-CONSOLE-23-005), DevOps manifest pipeline (`deploy/downloads/manifest.json`), Offline Kit metadata (`manifest/offline-manifest.json`), and export orchestrator `/console/exports`. +- **Feature flags:** `downloads.workspace.enabled`, `downloads.exportQueue`, `downloads.offlineParity`. +- **Tenancy:** Artefacts are tenant-agnostic except evidence bundles, which are tagged with originating tenant and require matching Authority scopes. + +--- + +## 2 - Workspace layout + +``` ++---------------------------------------------------------------+ +| Header: Snapshot timestamp - Manifest signature status | ++---------------------------------------------------------------+ +| Cards: Latest release - Offline kit parity - Export queue | ++---------------------------------------------------------------+ +| Tabs: Artefacts | Exports | Offline Kits | Webhooks | ++---------------------------------------------------------------+ +| Filter bar: Channel - Kind - Architecture - Scope tags | ++---------------------------------------------------------------+ +| Table (virtualised): Artifact | Channel | Digest | Status | +| Detail drawer: Metadata | Commands | Provenance | History | ++---------------------------------------------------------------+ +``` + +- **Snapshot banner:** shows `manifest.version`, `generatedAt`, and cosign verification state. If verification fails, the banner turns red and links to troubleshooting guidance. +- **Quick actions:** Copy manifest URL, download attestation bundle, trigger parity check, open CLI parity doc (`/docs/cli-vs-ui-parity.md`). +- **Filters:** allow narrowing by channel (`edge`, `stable`, `airgap`), artefact kind (`container.image`, `helm.chart`, `compose.bundle`, `offline.bundle`, `export.bundle`), architecture (`linux/amd64`, `linux/arm64`), and scope tags (`console`, `scheduler`, `authority`). + +--- + +## 3 - Artefact catalogue + +| Category | Artefacts surfaced | Source | Notes | +|----------|-------------------|--------|-------| +| **Core containers** | `stellaops/web-ui`, `stellaops/web`, `stellaops/concelier`, `stellaops/excititor`, `stellaops/scanner-*`, `stellaops/authority`, `stellaops/attestor`, `stellaops/scheduler-*` | `deploy/downloads/manifest.json` (`artifacts[].kind = "container.image"`) | Digest-only pulls with copy-to-clipboard `docker pull` and `oras copy` commands; badges show arch availability. | +| **Helm charts** | `deploy/helm/stellaops-*.tgz` plus values files | Manifest entries where `kind = "helm.chart"` | Commands reference `helm repo add` (online) and `helm install --values` (offline). UI links to values matrix in `/docs/install/helm-prod.md` when available. | +| **Compose bundles** | `deploy/compose/docker-compose.*.yaml`, `.env` seeds | `kind = "compose.bundle"` | Inline diff viewer highlights digest changes vs previous snapshot; `docker compose pull` command copies digest pins. | +| **Offline kit** | `stella-ops-offline-kit--.tar.gz` + signatures and manifest | Offline Kit metadata (`manifest/offline-manifest.json`) merged into downloads view | Drawer shows bundle size, signed manifest digest, cosign verification command (mirrors `/docs/24_OFFLINE_KIT.md`). | +| **Evidence exports** | Completed jobs from `/console/exports` (findings delta, policy explain, run evidence) | Export orchestrator job queue | Entries expire after retention window; UI exposes `stella runs export` and `stella findings export` parity buttons. | +| **Webhooks & parity** | `/downloads/hooks/subscribe` configs, CI parity reports | Manifest extras (`kind = "webhook.config"`, `kind = "parity.report"`) | Operators can download webhook payload templates and review the latest CLI parity check report generated by docs CI. | + +--- + +## 4 - Manifest structure + +The DevOps pipeline publishes a deterministic manifest at `deploy/downloads/manifest.json`, signed with the release Cosign key (`DOWNLOADS-CONSOLE-23-001`). The Console fetches it on workspace load and caches it with `If-None-Match` headers to avoid redundant pulls. The manifest schema: + +- **`version`** - monotonically increasing integer tied to pipeline run. +- **`generatedAt`** - ISO-8601 UTC timestamp. +- **`signature`** - URL to detached Cosign signature (`manifest.json.sig`). +- **`artifacts[]`** - ordered list keyed by `id`. + +Each artefact contains: + +| Field | Description | +|-------|-------------| +| `id` | Stable identifier (`::`). | +| `kind` | One of `container.image`, `helm.chart`, `compose.bundle`, `offline.bundle`, `export.bundle`, `webhook.config`, `parity.report`. | +| `channel` | `edge`, `stable`, or `airgap`. | +| `version` | Semantic or calendar version (for containers, matches release manifest). | +| `architectures` | Array of supported platforms (empty for arch-agnostic artefacts). | +| `digest` | SHA-256 for immutable artefacts; Compose bundles include file hash. | +| `sizeBytes` | File size (optional for export bundles that stream). | +| `downloadUrl` | HTTPS endpoint (registry, object store, or mirror). | +| `signatureUrl` | Detached signature (Cosign, DSSE, or attestation) if available. | +| `sbomUrl` | Optional SBOM pointer (CycloneDX JSON). | +| `attestationUrl` | Optional in-toto/SLSA attestation. | +| `docs` | Array of documentation links (e.g., `/docs/install/docker.md`). | +| `tags` | Free-form tags (e.g., `["console","ui","offline"]`). | + +### 4.1 Example excerpt + +```json +{ + "version": 42, + "generatedAt": "2025-10-27T04:00:00Z", + "signature": "https://downloads.stella-ops.org/manifest/manifest.json.sig", + "artifacts": [ + { + "id": "container.image:web-ui:2025.10.0-edge", + "kind": "container.image", + "channel": "edge", + "version": "2025.10.0-edge", + "architectures": ["linux/amd64", "linux/arm64"], + "digest": "sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf", + "sizeBytes": 187563210, + "downloadUrl": "https://registry.stella-ops.org/v2/stellaops/web-ui/manifests/sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf", + "signatureUrl": "https://downloads.stella-ops.org/signatures/web-ui-2025.10.0-edge.cosign.sig", + "sbomUrl": "https://downloads.stella-ops.org/sbom/web-ui-2025.10.0-edge.cdx.json", + "attestationUrl": "https://downloads.stella-ops.org/attestations/web-ui-2025.10.0-edge.intoto.jsonl", + "docs": ["/docs/install/docker.md", "/docs/security/console-security.md"], + "tags": ["console", "ui"] + }, + { + "id": "offline.bundle:ouk:2025.10.0-edge", + "kind": "offline.bundle", + "channel": "edge", + "version": "2025.10.0-edge", + "digest": "sha256:4f7d2f7a8d0cf4b5f3af689f6c74cd213f4c1b3a1d76d24f6f9f3d9075e51f90", + "downloadUrl": "https://downloads.stella-ops.org/offline/stella-ops-offline-kit-2025.10.0-edge.tar.gz", + "signatureUrl": "https://downloads.stella-ops.org/offline/stella-ops-offline-kit-2025.10.0-edge.tar.gz.sig", + "sbomUrl": "https://downloads.stella-ops.org/offline/offline-manifest-2025.10.0-edge.json", + "docs": ["/docs/24_OFFLINE_KIT.md"], + "tags": ["offline", "airgap"] + } + ] +} +``` + +Console caches the manifest hash and surfaces differences when a new version lands, helping operators confirm digests drift only when expected. + +--- + +## 5 - Download workflows and statuses + +| Status | Applies to | Behaviour | +|--------|------------|-----------| +| **Ready** | Immutable artefacts (images, Helm/Compose bundles, offline kit) | Commands available immediately. Digest, size, and last verification timestamp display in the table. | +| **Pending export** | Async exports queued via `/console/exports` | Shows job owner, scope, and estimated completion time. UI polls every 15 s and updates progress bar. | +| **Processing** | Long-running export (evidence bundle, large SBOM) | Drawer shows current stage (`collecting`, `compressing`, `signing`). Operators can cancel if they own the request and hold `downloads.manage`. | +| **Delivered** | Completed export within retention window | Provides download links, resume token, and parity snippet for CLI. | +| **Expired** | Export past retention or manually expired | Row grays out; clicking opens housekeeping guidance with CLI command to regenerate (`stella runs export --run `). | + +Exports inherit retention defaults defined in policy (`downloads.retentionDays`, min 3, max 30). Operators can override per tenant if they have the appropriate scope. + +--- + +## 6 - CLI parity and copy-to-clipboard + +- **Digest pulls:** Each container entry exposes `docker pull @` and `oras copy @ --to-dir ./downloads` buttons. Commands include architecture hints for multi-platform images. +- **Helm/Compose:** Buttons output `helm pull` / `helm install` with the manifest URL and `docker compose --env-file` commands referencing the downloaded bundle. +- **Offline kit:** Copy buttons produce the full verification sequence: + +```bash +curl -LO https://downloads.stella-ops.org/offline/stella-ops-offline-kit-2025.10.0-edge.tar.gz +curl -LO https://downloads.stella-ops.org/offline/stella-ops-offline-kit-2025.10.0-edge.tar.gz.sig +cosign verify-blob \ + --key https://stella-ops.org/keys/cosign.pub \ + --signature stella-ops-offline-kit-2025.10.0-edge.tar.gz.sig \ + stella-ops-offline-kit-2025.10.0-edge.tar.gz +``` + +- **Exports:** Drawer lists CLI equivalents (for example, `stella findings export --run `). When the CLI supports resume tokens, the command includes `--resume-token` from the manifest entry. +- **Automation:** Webhook tab copies `curl` snippets to subscribe to `/downloads/hooks/subscribe?topic=` and includes payload schema for integration tests. + +Parity buttons write commands to the clipboard and display a toast confirming scope hints (for example, `Requires downloads.read + tenant scope`). Accessibility shortcuts (`Shift+D`) trigger the primary copy action for keyboard users. + +--- + +## 7 - Offline and air-gap workflow + +- **Manifest sync:** Offline users download `manifest/offline-manifest.json` plus detached JWS and import it via `stella offline kit import`. Console highlights if the offline manifest predates the online manifest by more than 7 days. +- **Artefact staging:** The workspace enumerates removable media instructions (export to `./staging//`) and warns when artefacts exceed configured media size thresholds. +- **Mirrors:** Buttons copy `oras copy` commands that mirror images to an internal registry (`registry..internal`). Operators can toggle `--insecure-policy` if the destination uses custom trust roots. +- **Parity checks:** `downloads.offlineParity` flag surfaces the latest parity report verifying that Offline Kit contents match the downloads manifest digests. If diff detected, UI raises a banner linking to remediation steps. +- **Audit logging:** Every download command triggered from the UI emits `ui.download.commandCopied` with artifact ID, digest, and tenant. Logs feed the evidence locker so air-gap imports can demonstrate provenance. + +--- + +## 8 - Observability and quotas + +| Signal | Source | Description | +|--------|--------|-------------| +| `ui_download_manifest_refresh_seconds` | Console metrics | Measures time to fetch and verify manifest. Targets < 3 s. | +| `ui_download_export_queue_depth` | `/console/downloads` API | Number of pending exports (per tenant). Surfaces as card and Grafana panel. | +| `ui_download_command_copied_total` | Console logs | Count of copy actions by artifact type, used to gauge CLI parity adoption. | +| `downloads.export.duration` | Export orchestrator | Duration histograms for bundle generation; alerts if P95 > 60 s. | +| `downloads.quota.remaining` | Authority quota service | Anonymous users limited to 33 exports/day, verified users 333/day. Banner turns amber at 90 % usage as per platform policy. | + +Telemetry entries include correlation IDs that match backend manifest refresh logs and export job records to keep troubleshooting deterministic. + +--- + +## 9 - References + +- `/docs/ui/console-overview.md` - primary shell, tenant controls, SSE ticker. +- `/docs/ui/navigation.md` - route ownership and keyboard shortcuts. +- `/docs/ui/sbom-explorer.md` - export flows feeding the downloads queue. +- `/docs/ui/runs.md` - evidence bundle integration. +- `/docs/24_OFFLINE_KIT.md` - offline kit packaging and verification. +- `/docs/security/console-security.md` - scopes, CSP, and download token handling (pending). +- `/docs/cli-vs-ui-parity.md` - CLI equivalence checks (pending). +- `deploy/releases/*.yaml` - source of container digests mirrored into the manifest. + +--- + +## 10 - Compliance checklist + +- [ ] Manifest schema documented (fields, signature, caching) and sample kept current. +- [ ] Artefact categories mapped to manifest entries and parity workflows. +- [ ] Download statuses, retention, and cancellation rules explained. +- [ ] CLI copy-to-clipboard commands mirror console actions with scope hints. +- [ ] Offline/air-gap parity workflow, mirror commands, and audit logging captured. +- [ ] Observability metrics and quota signalling documented. +- [ ] References cross-linked to adjacent docs (navigation, exports, offline kit). +- [ ] Accessibility shortcuts and copy-to-clipboard behaviour noted with compliance reminder. + +--- + +*Last updated: 2025-10-27 (Sprint 23).* diff --git a/docs/ui/findings.md b/docs/ui/findings.md new file mode 100644 index 00000000..776b6713 --- /dev/null +++ b/docs/ui/findings.md @@ -0,0 +1,179 @@ +# StellaOps Console - Findings + +> **Audience:** Policy Guild, Console UX team, security analysts, customer enablement. +> **Scope:** Findings list UX, filters, saved views, explain drawer, exports, CLI parity, real-time updates, and offline considerations for Sprint 23. + +The Findings workspace visualises materialised policy verdicts produced by the Policy Engine. It lets analysts triage affected components, inspect explain traces, compare policy views, and export evidence while respecting Aggregation-Only guardrails. + +--- + +## 1. Access and prerequisites + +- **Route:** `/console/findings` with optional panel parameters (e.g., `/console/findings?panel=explain&finding=`). +- **Scopes:** `findings.read` (list), `policy:runs` (view run metadata), `policy:simulate` (stage simulations), `downloads.read` (export bundles). +- **Prerequisites:** Policy Engine v2 (`policy_run` and `effective_finding_*` endpoints), Concelier/Excititor feeds for provenance, SBOM Service for component metadata. +- **Feature flags:** `findings.explain.enabled`, `findings.savedViews.enabled`, `findings.simulationDiff.enabled`. +- **Tenancy:** All queries include tenant context; cross-tenant comparisons require explicit admin scope and render split-pane view. + +--- + +## 2. Layout overview + +``` ++-------------------------------------------------------------------+ +| Header: Tenant badge - policy selector - global filters - actions | ++-------------------------------------------------------------------+ +| Top row cards: Affected assets - Critical count - KEV count | ++-------------------------------------------------------------------+ +| Findings grid (virtualised) | +| Columns: Status | Severity | Component | Policy | Source | Age | +| Row badges: KEV, Quieted, Override, Simulation only | ++-------------------------------------------------------------------+ +| Right drawer / full view tabs: Summary | Explain | Evidence | Run | ++-------------------------------------------------------------------+ +``` + +The policy selector includes Active, Staged, and Simulation snapshots. Switching snapshots triggers diff banners to highlight changes. + +--- + +## 3. Filters and saved views + +| Filter | Description | Notes | +|--------|-------------|-------| +| **Status** | `affected`, `at_risk`, `quieted`, `fixed`, `not_applicable`, `mitigated`. | Status definitions align with Policy Engine taxonomy. | +| **Severity** | Critical, High, Medium, Low, Informational, Untriaged. | Derived from policy scoring; UI displays numeric score tooltip. | +| **KEV** | Toggle to show only Known Exploited Vulnerabilities. | Pulls from Concelier enrichment. | +| **Policy** | Active, Staged, Simulation snapshots. | Simulation requires recent run; otherwise greyed out. | +| **Component** | PURL or substring search. | Autocomplete draws from current tenant findings. | +| **SBOM** | Filter by image digest or SBOM ID. | Includes quick links to SBOM Explorer. | +| **Tag** | Team or environment tags emitted by Policy Engine (`tags[]`). | Supports multi-select. | +| **Run window** | `Last 24h`, `Last 7d`, `Custom range`. | Applies to run timestamp. | +| **Explain hints** | Filter by explain artefact (rule ID, justification, VEX provider). | Uses server-side filter parameters. | + +Saved views persist filter combinations per tenant and policy. Users can mark views as shared; shared views appear in the left rail with owner and last updated timestamp. Keyboard shortcuts align with global presets (`Cmd+1-9 / Ctrl+1-9`). + +--- + +## 4. Findings grid + +| Column | Details | +|--------|---------| +| **Status** | Badge with tooltip describing resolution path (e.g., "Affected - blocked by policy rule R-105"). Quieted findings show a muted badge with expiry. | +| **Severity** | Numeric score and label. Hover reveals scoring formula and evidence sources. | +| **Component** | PURL plus human-friendly name. Includes SBOM badge linking to SBOM Explorer detail. | +| **Policy** | Policy name + revision digest; clicking opens policy diff in new tab. | +| **Source signals** | Icons for VEX, Advisory, Runtime overlays. Hover shows counts and last updated timestamps. | +| **Age** | Time since finding was last evaluated; colour-coded when exceeding SLA. | + +Row indicators: + +- **KEV** badge when Concelier marks the vulnerability as exploited. +- **Override** badge when policy override or exemption applied. +- **Simulation only** badge when viewing simulation snapshot; warns that finding is not yet active. +- **Determinism alert** icon if latest run reported a determinism mismatch (links to run detail). + +Bulk actions (multi-select): + +- `Open explains` (launch explain drawer for up to 10 findings). +- `Export CSV/JSON`. +- `Copy CLI` commands for batch explains (`stella findings explain --batch file`). +- `Create ticket` (integrates with integrations configured under Admin). + +--- + +## 5. Explain drawer + +Tabs inside the explain drawer: + +1. **Summary** - status, severity, policy decision, rule ID, last evaluated timestamp, SBOM link, run ID. +2. **Rule chain** - ordered list of policy rules triggered; each entry shows rule ID, name, action (block/warn/quiet), score contribution, and condition snippet. +3. **Evidence** - references to Concelier advisories, Excititor consensus, runtime signals, and overrides. Evidence entries link to their respective explorers. +4. **VEX impact** - table of VEX claims considered; displays provider, status, justification, acceptance (accepted/ignored), weight. +5. **History** - timeline of state transitions (affected -> quieted -> mitigated) with timestamps and operators (if override applied). +6. **Raw trace** - canonical JSON trace from Policy Engine (read-only). CLI parity snippet: + - `stella findings explain --policy --finding --format json`. + +Explain drawer includes copy-to-clipboard buttons for rule chain and evidence JSON to support audit workflows. When sealed mode is active, a banner highlights which evidence was sourced from cached data. + +--- + +## 6. Simulations and comparisons + +- Simulation toggle lets analysts compare Active vs Staged/Sandbox policies. +- Diff banner summarises added, removed, and changed findings. +- Side-by-side view shows baseline vs simulation verdicts with change badges (`added`, `removed`, `severity up`, `severity down`). +- CLI parity callout: `stella policy simulate --policy --sbom --format diff`. +- Simulation results persist for 7 days; stale simulations prompt re-run recommendation. + +--- + +## 7. Exports and automation + +- Immediate exports: CSV, JSON, Markdown summary for selected findings. +- Scheduled exports: asynchronous job to generate full tenant report (JSON + CSV) with manifest digests. +- Explain bundle export packages traces for a set of findings; includes manifest and hash for offline review. +- CLI parity: + - `stella findings ls --policy --format json --output findings.json` + - `stella findings export --policy --format csv --output findings.csv` + - `stella findings explain --batch batch.txt --output explains/` +- Automation: webhook copy button for `/downloads/hooks/subscribe?topic=findings.report.ready`. + +--- + +## 8. Real-time updates and observability + +- SSE channel `/console/findings/stream` pushes new findings, status changes, and quieted expirations; UI animates affected rows. +- Header cards show metrics: `findings_critical_total`, `findings_quieted_total`, `findings_kev_total`. +- Run ticker lists latest policy runs with status, duration, determinism hash. +- Error banners include correlation IDs linking to Policy Engine run logs. +- Metrics drill-down links to dashboards (OpenTelemetry, Prometheus). + +--- + +## 9. Offline and air-gap behaviour + +- Offline banner indicates snapshot ID and timestamp used for findings. +- Explain drawer notes when evidence references offline bundles; suggests importing updated advisories/VEX to refresh results. +- Exports default to local storage paths; UI provides manual transfer instructions. +- CLI examples switch to include `--sealed` or `--offline` flags. +- Tenant selector hides tenants without corresponding offline findings data to avoid partial views. + +--- + +## 10. Screenshot coordination + +- Placeholders: + - `![Findings grid placeholder](../assets/ui/findings/grid-placeholder.png)` + - `![Explain drawer placeholder](../assets/ui/findings/explain-placeholder.png)` +- Coordinate with Console Guild (Slack `#console-screenshots`, entry 2025-10-26) to capture updated light and dark theme shots before release. + +--- + +## 11. References + +- `/docs/ui/console-overview.md` - shell, filters, tenant model. +- `/docs/ui/navigation.md` - route list, deep-link schema. +- `/docs/ui/advisories-and-vex.md` - advisory and VEX context feeding findings. +- `/docs/ui/policies.md` (pending) - editor and policy lifecycle. +- `/docs/policy/overview.md` - Policy Engine outputs. +- `/docs/policy/runs.md` - run orchestration. +- `/docs/cli/policy.md` - CLI parity for findings commands. + +--- + +## 12. Compliance checklist + +- [ ] Filters and saved view behaviour documented with CLI alignment. +- [ ] Findings grid columns, badges, and bulk actions captured. +- [ ] Explain drawer walkthrough includes rule chain, evidence, and raw trace. +- [ ] Simulation diff behaviour and CLI callouts described. +- [ ] Exports (immediate and scheduled) plus webhook integration covered. +- [ ] Real-time updates, metrics, and error correlation documented. +- [ ] Offline behaviour and screenshot coordination noted. +- [ ] References validated. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* + diff --git a/docs/ui/navigation.md b/docs/ui/navigation.md new file mode 100644 index 00000000..066558ab --- /dev/null +++ b/docs/ui/navigation.md @@ -0,0 +1,163 @@ +# StellaOps Console - Navigation + +> **Audience:** Console UX writers, UI engineers, QA, and enablement teams. +> **Scope:** Primary route map, layout conventions, keyboard shortcuts, deep-link patterns, and tenant context switching for the StellaOps Console (Sprint 23). + +The navigation framework keeps Console workflows predictable across tenants and deployment modes. This guide explains how the global shell, feature routes, and context tokens cooperate so operators can jump between findings, SBOMs, advisories, policies, and runs without losing scope. + +--- + +## 1. Information Architecture + +### 1.1 Primary routes + +| Route pattern | Module owner | Purpose | Required scopes (minimum) | Core services | +|---------------|--------------|---------|---------------------------|---------------| +| `/console/dashboard` | Web gateway | Landing KPIs, feed age, queue depth, alerts | `ui.read` | Web, Scheduler WebService, Concelier WebService, Excititor WebService | +| `/console/findings` | Policy Engine | Aggregated findings, explain drawer, export | `findings.read` | Policy Engine, Concelier WebService, SBOM Service | +| `/console/sbom` | SBOM Service | Catalog view, component graph, overlays | `sbom.read` | SBOM Service, Policy Engine (overlays) | +| `/console/advisories` | Concelier | Advisory aggregation with provenance banners | `advisory.read` | Concelier WebService | +| `/console/vex` | Excititor | VEX aggregation, consensus, conflicts | `vex.read` | Excititor WebService | +| `/console/runs` | Scheduler | Run list, live progress, evidence downloads | `runs.read` | Scheduler WebService, Policy Engine, Scanner WebService | +| `/console/policies` | Policy Engine | Editor, simulations, approvals | `policy.read` (read) / `policy.write` (edit) | Policy Engine, Authority | +| `/console/downloads` | DevOps | Signed artifacts, Offline Kit parity checklist | `downloads.read` | DevOps manifest API, Offline Kit | +| `/console/admin` | Authority | Tenants, roles, tokens, integrations | `ui.admin` (plus scoped `authority:*`) | Authority | +| `/console/help` | Docs Guild | Guides, tours, release notes | `ui.read` | Docs static assets | + +### 1.2 Secondary navigation elements + +- **Left rail:** highlights the active top-level route, exposes quick metrics, and shows pinned saved views. Keyboard focus cycles through rail entries with `Tab`/`Shift+Tab`. +- **Breadcrumb bar:** renders `Home / Module / Detail` format. Detail crumbs include IDs and titles for shareable context (for example, `Findings / High Severity / CVE-2025-1234`). +- **Action shelf:** right-aligned controls for context actions (export, verify, retry). Buttons disable automatically if the current subject lacks the requisite scope. + +--- + +## 2. Command Palette and Search + +- **Trigger:** `Ctrl/Cmd + K`. Palette opens in place, keeps focus, and announces results via ARIA live region. +- **Capabilities:** jump to routes, saved views, tenants, recent entities (findings, SBOMs, advisories), and command actions (for example, "Start verification", "Open explain drawer"). +- **Result tokens:** palette entries carry metadata (`type`, `tenant`, `filters`). Selecting an item updates the URL and applies stored filters without a full reload. +- **Offline fallback:** in sealed/offline mode, palette restricts actions to cached routes and saved views; remote-only items show a grayed-out badge. + +--- + +## 3. Global Filters and Context Chips + +| Control | Shortcut | Persistence | Notes | +|---------|----------|-------------|-------| +| **Tenant picker** | `Ctrl/Cmd + T` | SessionStorage + URL `tenant` query | Issues fresh Authority token, invalidates caches, emits `ui.tenant.switch` log. | +| **Filter tray** | `Shift + F` | IndexedDB (per tenant) + URL query (`since`, `severity`, `tags`, `source`, `status`, `policyView`) | Applies instantly to compatible routes; incompatible filters show a reset suggestion. | +| **Component search** | `/` when filters closed | URL `component` query | Context-aware; scopes results to current tenant and module. | +| **Time window** | `Ctrl/Cmd + Shift + 1-4` | URL `since`/`until`, palette preset | Mapped to preset windows: 24 h, 7 d, 30 d, custom. | + +Context chips appear beneath page titles summarising active filters (for example, `Tenant: west-prod`, `Severity: Critical+High`, `Time: Last 7 days`). Removing a chip updates the tray and URL atomically. + +--- + +## 4. Keyboard Shortcut Matrix + +| Scope | Shortcut (Mac / Windows) | Action | Notes | +|-------|--------------------------|--------|-------| +| Global | `Cmd+K / Ctrl+K` | Open command palette | Accessible from any route except modal dialogs. | +| Global | `Cmd+T / Ctrl+T` | Open tenant switcher | Requires `ui.read`. Confirm selection with `Enter`; `Esc` cancels without switching. | +| Global | `Shift+F` | Toggle global filter tray | Focus lands on first filter control. | +| Global | `Cmd+1-9 / Ctrl+1-9` | Load saved view preset | Each preset bound per tenant; non-assigned keys show tooltip. | +| Global | `?` | Show keyboard reference overlay | Overlay lists context-specific shortcuts; closes with `Esc`. | +| Findings module | `Cmd+/ / Ctrl+/` | Focus explain search | Works when explain drawer is open. | +| SBOM module | `Cmd+G / Ctrl+G` | Toggle graph overlays | Persists per session. | +| Advisories & VEX | `Cmd+Opt+F / Ctrl+Alt+F` | Focus provider filter | Highlights provider chip strip. | +| Runs module | `Cmd+R / Ctrl+R` | Refresh SSE snapshot | Schedules soft refresh (no hard reload). | +| Policies module | `Cmd+S / Ctrl+S` | Save draft (if edit rights) | Mirrors Policy Editor behaviour. | + +Shortcut handling follows WCAG 2.2 best practices: all accelerators are remappable via Settings -> Accessibility -> Keyboard shortcuts, and the overlay documents platform differences. + +--- + +## 5. Deep-Link Patterns + +### 5.1 URL schema + +Console URLs adopt the format: + +``` +/console/[/:id][/:tab]?tenant=&since=&severity=&view=&panel=&component= +``` + +- **`tenant`** is mandatory and matches Authority slugs (e.g., `acme-prod`). +- **`since` / `until`** use ISO-8601 timestamps (UTC). Preset ranges set only `since`; UI computes `until` on load. +- **`severity`** accepts comma-separated policy buckets (e.g., `critical,high,kev`). +- **`view`** stores module-specific state (e.g., `sbomView=usage`, `findingsPreset=threat-hunting`). +- **`panel`** selects drawers or tabs (`panel=explain`, `panel=timeline`). + +### 5.2 Copyable links + +- Share links from the action shelf or context chips; both copy canonical URLs with all active filters. +- CLI parity: inline callouts provide `stella` commands derived from the URL parameters to ensure console/CLI equivalence. +- Offline note: links copied in sealed mode include the snapshot ID (`snapshot=`) so recipients know which offline data set to load. + +### 5.3 Examples + +- **`since` / `until`** use ISO-8601 timestamps (UTC). Preset ranges set only `since`; UI computes `until` on load. +- **`severity`** accepts comma-separated policy buckets (e.g., `critical,high,kev`). +- **`view`** stores module-specific state (e.g., `sbomView=usage`, `findingsPreset=threat-hunting`). +- **`panel`** selects drawers or tabs (`panel=explain`, `panel=timeline`). +- **`component`** encodes package selection using percent-encoded PURL syntax. +- **`snapshot`** appears when copying links offline to reference Offline Kit build hash. +@@ +| Use case | Example URL | Description | +|----------|-------------|-------------| +| Findings triage | `/console/findings?v=table&severity=critical,high&tenant=west-prod&since=2025-10-20T00:00:00Z` | Opens the findings table limited to critical/high for west-prod, last 7 days. | +| SBOM component focus | `/console/sbom/sha256:abcd?tenant=west-prod&component=pkg:npm/react@18.3.0&view=usage` | Deep-links to a specific image digest and highlights an NPM package in Usage view. | +| Advisory explain | `/console/advisories?tenant=west-prod&source=nvd&panel=detail&documentId=CVE-2025-1234` | Opens advisory list filtered to NVD and expands CVE detail drawer. | +| Run monitor | `/console/runs/42?tenant=west-prod&panel=progress` | Focuses run ID 42 with progress drawer active (SSE stream attached). | + +--- + +## 6. Tenant Switching Lifecycle + +1. **Initiate:** User triggers `Ctrl/Cmd + T` or clicks the tenant badge. Switcher modal lists authorised tenants and recent selections. +2. **Preview:** Selecting a tenant shows summary (environment, last snapshot, role coverage). The modal flags tenants missing required scopes for the current route. +3. **Confirm:** On confirmation, the UI requests a new DPoP-bound access token from Authority (`aud=console`, `tenant=`). +4. **Invalidate caches:** Stores keyed by tenant purge automatically; modules emit `tenantChanged` events so in-flight SSE streams reconnect with new headers. +5. **Restore state:** Global filters reapply where valid. Incompatible filters (for example, a saved view unavailable in the new tenant) prompt users to pick a fallback. +6. **Audit and telemetry:** `ui.tenant.switch` log writes subject, from/to tenant, correlation ID. Metric `ui_tenant_switch_total` increments for observability dashboards. +7. **Offline behaviour:** If the target tenant is absent from the offline snapshot, switcher displays guidance to import updated Offline Kit data before proceeding. + +--- + +## 7. Breadcrumbs, Tabs, and Focus Management + +- Breadcrumb titles update synchronously with route data loads. When fragments change (for example, selecting a finding), the breadcrumb text updates without pushing a new history entry to keep back/forward predictable. +- Detail views rely on accessible tabs (`role="tablist"`) with keyboard support (`ArrowLeft/Right`). Tab selection updates the URL `tab` parameter for deep linking. +- Focus management: + - Route changes send focus to the primary heading (`h1`) using the live region announcer. + - Opening drawers or modals traps focus until closed; ESC returns focus to the triggering element. + - Keyboard-only navigation is validated via automated Playwright accessibility checks as part of `DEVOPS-CONSOLE-23-001`. + +--- + +## 8. References + +- `/docs/ui/console-overview.md` - structural overview, tenant model, global filters. +- `/docs/ui/sbom-explorer.md` - SBOM-specific navigation and graphs (pending). +- `/docs/ui/advisories-and-vex.md` - aggregation UX details (pending). +- `/docs/ui/findings.md` - findings filters and explain drawer (pending). +- `/docs/security/console-security.md` - Authority, scopes, CSP. +- `/docs/cli-vs-ui-parity.md` - CLI equivalence matrix. +- `/docs/accessibility.md` - keyboard remapping, WCAG validation checklists. + +--- + +## 9. Compliance Checklist + +- [ ] Route table matches Console build (paths, scopes, owners verified with Console Guild). +- [ ] Keyboard shortcut matrix reflects implemented accelerators and accessibility overlay. +- [ ] Deep-link examples tested for copy/share parity and CLI alignment. +- [ ] Tenant switching flow documents cache invalidation and audit logging. +- [ ] Filter tray, command palette, and presets cross-referenced with accessibility guidance. +- [ ] Offline/air-gap notes included for palette, tenant switcher, and deep-link metadata. +- [ ] Links to dependent docs (`/docs/ui/*`, `/docs/security/*`) validated. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* diff --git a/docs/ui/policies.md b/docs/ui/policies.md new file mode 100644 index 00000000..a1dc7687 --- /dev/null +++ b/docs/ui/policies.md @@ -0,0 +1,191 @@ +# StellaOps Console - Policies Workspace + +> **Audience:** Policy Guild, Console UX, product ops, review leads. +> **Scope:** Policy workspace navigation, editor surfaces, simulation, approvals, RBAC, observability, offline behaviour, and CLI parity for Sprint 23. + +The Policies workspace centralises authoring, simulation, review, and promotion for `stella-dsl@1` packs. It builds on the Policy Editor (`docs/ui/policy-editor.md`) and adds list views, governance workflows, and integrations with runs and findings. + +--- + +## 1. Access and prerequisites + +- **Routes:** + - `/console/policies` (list) + - `/console/policies/:policyId` (details) + - `/console/policies/:policyId/:revision` (editor, approvals, runs) +- **Scopes:** + - `policy:read` (list and details) + - `policy:write` (edit drafts, run lint/compile) + - `policy:submit`, `policy:review`, `policy:approve` (workflow actions) + - `policy:runs` (view run history) + - `policy:simulate` (run simulations) + - `effective:write` (promotion visibility only; actual write remains server-side) +- **Feature flags:** `policy.studio.enabled`, `policy.simulation.diff`, `policy.runCharts.enabled`, `policy.offline.bundleUpload`. +- **Dependencies:** Policy Engine v2 APIs (`/policies`, `/policy/runs`, `/policy/simulations`), Policy Studio Monaco assets, Authority fresh-auth flows for critical operations. + +--- + +## 2. List and detail views + +### 2.1 Policy list + +| Column | Description | +|--------|-------------| +| **Policy** | Human-readable name plus policy ID (e.g., `P-7`). | +| **State** | `Active`, `Draft`, `Staged`, `Simulation`, `Archived`. Badge colours align with Policy Engine status. | +| **Revision** | Latest revision digest (short SHA). | +| **Owner** | Primary maintainer or owning team tag. | +| **Last change** | Timestamp and actor of last update (edit, submit, approve). | +| **Pending approvals** | Count of outstanding approval requests (with tooltip listing reviewers). | + +Row actions: `Open`, `Duplicate`, `Export pack`, `Run simulation`, `Compare revisions`. + +Filters: owning team, state, tag, pending approvals, contains staged changes, last change window, simulation warnings (determinism, failed run). + +### 2.2 Policy detail header + +- Summary cards: current state, digest, active revision, staged revision (if any), simulation status, last production run (timestamp, duration, determinism hash). +- Action bar: `Edit draft`, `Run simulation`, `Submit for review`, `Promote`, `Export pack`, `View findings`. + +--- + +## 3. Editor shell + +The editor view reuses the structure documented in `/docs/ui/policy-editor.md` and adds: + +- **Context banner** showing tenant, policy ID, revision digest, and simulation badge if editing sandbox copy. +- **Lint and compile status** displayed inline with time since last run. +- **Checklist sidebar** summarising required steps (lint pass, simulation run, deterministic CI, security review). Each item links to evidence (e.g., latest simulation diff). +- **Monaco integration** with policy-specific snippets, schema hover, code actions (`Insert allowlist`, `Add justification`). +- **Draft autosave** every 30 seconds with conflict detection (merges disabled; last write wins with warning). + +--- + +## 4. Simulation workflows + +- Simulation modal accepts SBOM filter (golden set, specific SBOM IDs, tenant-wide) and options for VEX weighting overrides. +- Simulations run asynchronously; progress shown in run ticker with status updates. +- Diff view summarises totals: affected findings added/removed, severity up/down counts, quieted changes. +- Side-by-side diff (Active vs Simulation) accessible directly from policy detail. +- Export options: JSON diff, Markdown summary, CLI snippet `stella policy simulate --policy --sbom `. +- Simulation results cached per draft revision. Cache invalidates when draft changes or SBOM snapshot updates. +- Simulation compliance card requires at least one up-to-date simulation before submission. + +--- + +## 5. Review and approval + +- **Review requests:** Authors tag reviewers; review sidebar lists pending reviewers, due dates, and escalation contact. +- **Comments:** Threaded comments support markdown, mentions, and attachments (redacted before persistence). Comment resolution required before approval. +- **Approval checklist:** + - Lint/compile success + - Simulation fresh (within configured SLA) + - Determinism verification passed + - Security review (if flagged) + - Offline bundle prepared (optional) +- **Fresh-auth:** Approve/promote buttons require fresh authentication; modal prompts for credentials and enforces short-lived token (<5 minutes). +- **Approval audit:** Approval events recorded with correlation ID, digests, reviewer note, effective date, and optional ticket link. + +--- + +## 6. Promotion and rollout + +- Promotion dialog summarises staged changes, target tenants, release windows, and run plan (full vs incremental). +- Operators can schedule promotion or apply immediately. +- Promotion triggers Policy Engine to materialise new revision; console reflects status and shows run progress. +- CLI parity: `stella policy promote --policy --revision --run-mode full`. +- Rollback guidance accessible from action bar (`Open rollback instructions`) linking to CLI command and documentation. + +--- + +## 7. Runs and observability + +- Runs tab displays table of recent runs with columns: run ID, type (`full`, `incremental`, `simulation`), duration, determinism hash, findings delta counts, triggered by. +- Charts: findings trend, quieted findings trend, rule hit heatmap (top rules vs recent runs). +- Clicking a run opens run detail drawer showing inputs (policy digest, SBOM batch hash, advisory snapshot hash), output summary, and explain bundle download. +- Error runs display red badge; detail drawer includes correlation ID and link to Policy Engine logs. +- SSE updates stream run status changes to keep UI real-time. + +--- + +## 8. RBAC and governance + +| Role | Scopes | Capabilities | +|------|--------|--------------| +| **Author** | `policy:read`, `policy:write`, `policy:simulate` | Create drafts, run lint/simulations, comment. | +| **Reviewer** | `policy:read`, `policy:review`, `policy:simulate` | Leave review comments, request changes. | +| **Approver** | `policy:read`, `policy:approve`, `policy:runs`, `policy:simulate` | Approve/promote, trigger runs, view run history. | +| **Operator** | `policy:read`, `policy:runs`, `policy:simulate`, `effective:write` | Schedule promotions, monitor runs (no editing). | +| **Admin** | Above plus Authority admin scopes | Manage roles, configure escalation chains. | + +UI disables controls not allowed by current scope and surfaces tooltip with required scope names. Audit log captures denied attempts (`policy.ui.action_denied`). + +--- + +## 9. Exports and offline bundles + +- `Export pack` button downloads policy pack (zip) with metadata, digest manifest, and README. +- Offline bundle uploader allows importing reviewed packs; UI verifies signatures and digests before applying. +- Explain bundle export collects latest run explain traces for audit. +- CLI parity: + - `stella policy export --policy --revision ` + - `stella policy bundle import --file ` + - `stella policy bundle export --policy --revision ` +- Offline mode displays banner and disables direct promotion; provides script instructions for offline runner. + +--- + +## 10. Observability and alerts + +- Metrics cards show `policy_run_seconds`, `policy_rules_fired_total`, `policy_determinism_failures_total`. +- Alert banners surfaced for determinism failures, simulation stale warnings, approval SLA breaches. +- Links to dashboards (Grafana) pre-filtered with policy ID. +- Telemetry panel lists last emitted events (policy.promoted, policy.simulation.completed). + +--- + +## 11. Offline and air-gap considerations + +- In sealed mode, editor warns about cached enrichment data; simulation run button adds tooltip explaining degraded evidence. +- Promotions queue and require manual CLI execution on authorised host; UI provides downloadable job manifest. +- Run charts switch to snapshot data; SSE streams disabled, replaced by manual refresh button. +- Export/download buttons label file paths for removable media transfer. + +--- + +## 12. Screenshot coordination + +- Placeholders: + - `![Policy list placeholder](../assets/ui/policies/list-placeholder.png)` + - `![Policy approval placeholder](../assets/ui/policies/approval-placeholder.png)` + - `![Simulation diff placeholder](../assets/ui/policies/simulation-placeholder.png)` +- Coordinate with Console Guild via `#console-screenshots` (entry 2025-10-26) to replace placeholders once UI captures are ready (light and dark themes). + +--- + +## 13. References + +- `/docs/ui/policy-editor.md` - detailed editor mechanics. +- `/docs/ui/findings.md` - downstream findings view and explain drawer. +- `/docs/policy/overview.md` and `/docs/policy/runs.md` - Policy Engine contracts. +- `/docs/security/authority-scopes.md` - scope definitions. +- `/docs/cli/policy.md` - CLI commands for policy management. +- `/docs/ui/console-overview.md` - navigation shell and filters. + +--- + +## 14. Compliance checklist + +- [ ] Policy list and detail workflow documented (columns, filters, actions). +- [ ] Editor shell extends Policy Studio guidance with checklists and lint/simulation integration. +- [ ] Simulation flow, diff presentation, and CLI parity captured. +- [ ] Review, approval, and promotion workflows detailed with scope gating. +- [ ] Runs dashboard, metrics, and SSE behaviour described. +- [ ] Exports and offline bundle handling included. +- [ ] Offline/air-gap behaviour and screenshot coordination recorded. +- [ ] References validated. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* + diff --git a/docs/ui/policy-editor.md b/docs/ui/policy-editor.md new file mode 100644 index 00000000..a997d9a2 --- /dev/null +++ b/docs/ui/policy-editor.md @@ -0,0 +1,178 @@ +# Policy Editor Workspace + +> **Audience:** Product/UX, UI engineers, policy authors/reviewers using the Console. +> **Scope:** Layout, features, RBAC, a11y, simulation workflow, approvals, run dashboards, and offline considerations for the Policy Engine v2 editor (“Policy Studio”). + +The Policy Editor is the primary Console workspace for composing, simulating, and approving `stella-dsl@1` policies. It combines Monaco-based editing, diff visualisations, and governance tools so authors and reviewers can collaborate without leaving the browser. + +--- + +## 1 · Access & Prerequisites + +- **Routes:** `/console/policy` (list) → `/console/policy/:policyId/:version?`. +- **Scopes:** + - `policy:write` to edit drafts, run lint/compile, attach simulations. + - `policy:submit` / `policy:review` / `policy:approve` for workflow actions. + - `policy:run` to trigger runs, `policy:runs` to inspect history. + - `findings:read` to open explain drawers. +- **Feature flags:** `policyStudio.enabled` (defaults true once Policy Engine v2 API available). +- **Browser support:** Evergreen Chrome, Edge, Firefox, Safari (last two versions). Uses WASM OPA sandbox; ensure COOP/COEP enabled per [UI architecture](../ARCHITECTURE_UI.md). + +--- + +## 2 · Workspace Layout + +``` +┌────────────────────────────────────────────────────────────────────────────┐ +│ Header: Policy selector • tenant switch • last activation banner │ +├────────────────────────────────────────────────────────────────────────────┤ +│ Sidebar (left) │ Main content (right) │ +│ - Revision list │ ┌───────────── Editor tabs ───────────────┐ │ +│ - Checklist status │ │ DSL │ Simulation │ Approvals │ ... │ │ +│ - Pending reviews │ └─────────────────────────────────────────┘ │ +│ - Run backlog │ │ +│ │ Editor pane / Simulation diff / Run viewer │ +└────────────────────────────────────────────────────────────────────────────┘ +``` + +- **Sidebar:** Revision timeline (draft, submitted, approved), compliance checklist cards, outstanding review requests, run backlog (incremental queue depth and SLA). +- **Editor tabs:** + - *DSL* (primary Monaco editor) + - *Simulation* (pre/post diff charts) + - *Approvals* (comments, audit log) + - *Runs* (heatmap dashboards) + - *Explain Explorer* (optional drawer for findings) +- **Right rail:** context cards for VEX providers, policy metadata, quick links to CLI/API docs. + +> Placeholder screenshot: `![Policy editor workspace](../assets/policy-editor/workspace.png)` (add after UI team captures latest build). + +--- + +## 3 · Editing Experience + +- Monaco editor configured for `stella-dsl@1`: + - Syntax highlighting, IntelliSense for rule/action names, snippets for common patterns. + - Inline diagnostics sourced from `/policies/{id}/lint` and `/compile`. + - Code actions (“Fix indentation”, “Insert requireVex block”). + - Mini-map disabled by default to reduce contrast noise; toggle available. +- **Keyboard shortcuts (accessible via `?`):** + - `Ctrl/Cmd + S` – Save draft (uploads to API if changed). + - `Ctrl/Cmd + Shift + Enter` – Run lint + compile. + - `Ctrl/Cmd + Shift + D` – Open diff view vs baseline. + - `Alt + Shift + F` – Format document (canonical ordering). +- **Schema tooltips:** Hover on `profile`, `rule`, `action` to view documentation (sourced from DSL doc). +- **Offline warnings:** When `sealed` mode detected, banner reminds authors to validate with offline bundle. + +--- + +## 4 · Simulation & Diff Panel + +- Triggered via “Run simulation” (toolbar) or automatically after compile. +- Displays: + - **Summary cards:** total findings added/removed/unchanged; severity up/down counts. + - **Rule hit table:** top rules contributing to diffs with percentage change. + - **Component list:** virtualised table linking to explain drawer; supports filters (severity, status, VEX outcome). + - **Visualisations:** stacked bar chart (severity deltas), sparkline for incremental backlog impact. +- Supports run presets: + - `Golden SBOM set` (default) + - Custom SBOM selection (via multi-select and search) + - Import sample JSON from CLI (`Upload diff`). +- Diff export options: + - `Download JSON` (same schema as CLI output) + - `Copy as Markdown` for review comments +- Simulation results persist per draft version; history accessible via timeline. + +--- + +## 5 · Review & Approval Workflow + +- **Commenting:** Line-level comments anchored to DSL lines; global comments supported. Uses rich text (Markdown subset) with mention support (`@group/sec-reviewers`). +- **Resolution:** Approvers/reviewers can mark comment resolved; history preserved in timeline. +- **Approval pane:** + - Checklist (lint, simulation, determinism CI) with status indicators; links to evidence. + - Reviewer checklist (quorum, blocking comments). + - Approval button only enabled when checklist satisfied. +- **Audit log:** Chronological view of submit/review/approve/archive events with actor, timestamp, note, attachments. +- **RBAC feedback:** When user lacks permission, actions are disabled with tooltip referencing required scope(s). +- **Notifications:** Integration with Notifier—subscribe/unsubscribe from review reminders within panel. + +--- + +## 6 · Runs & Observability + +- **Run tab** consumes `/policy/runs` data: + - Heatmap of rule hits per run (rows = runs, columns = top rules). + - VEX override counter, suppressions, quieted findings metrics. + - Incremental backlog widget (queue depth vs SLA). + - Export CSV/JSON button. +- **Replay/Download:** For each run, actions to download sealed replay bundle or open CLI command snippet. +- **Alert banners:** + - Determinism mismatch (red) + - SLA breach (amber) + - Pending replay (info) + +--- + +## 7 · Explain & Findings Integration + +- Inline “Open in Findings” button for any diff entry; opens side drawer with explain trace (same schema as `/findings/*/explain`). +- Drawer includes: + - Rule sequence with badges (block/warn/quiet). + - VEX evidence and justification codes. + - Links to advisories (Concelier) and SBOM components. +- Copy-to-clipboard (JSON) and “Share permalink” features (permalinks encode tenant, policy version, component). + +--- + +## 8 · Accessibility & i18n + +- WCAG 2.2 AA: + - Focus order follows logical workflow; skip link available. + - All actionable icons paired with text or `aria-label`. + - Simulation charts include table equivalents for screen readers. +- Keyboard support: + - `Alt+1/2/3/4` to switch tabs. + - `Shift+?` toggles help overlay (with key map). +- Internationalisation: + - Translations sourced from `/locales/{lang}.json`. + - Date/time displayed using user locale via `Intl.DateTimeFormat`. +- Theming: + - Light/dark CSS tokens; Monaco theme syncs with overall theme. + - High-contrast mode toggled via user preferences. + +--- + +## 9 · Offline & Air-Gap Behaviour + +- When console operates in sealed enclave: + - Editor displays “Sealed mode” banner with import timestamp. + - Simulation uses cached SBOM/advisory/VEX data only; results flagged accordingly. + - “Export bundle” button packages draft + simulations for transfer. +- Approvals require local Authority; UI blocks actions if `policy:approve` scope absent due to offline token limitations. +- Run tab surfaces bundle staleness warnings (`policy_runs.inputs.env.sealed=true`). + +--- + +## 10 · Telemetry & Testing Hooks + +- User actions (simulate, submit, approve, activate) emit telemetry (`ui.policy.action` spans) with anonymised metadata. +- Console surfaces correlation IDs for lint/compile errors to ease support triage. +- Cypress/Playwright fixtures available under `ui/policy-editor/examples/`; docs should note to re-record after significant UI changes. + +--- + +## 11 · Compliance Checklist + +- [ ] **Lint integration:** Editor surfaces diagnostics from API compile endpoint; errors link to DSL documentation. +- [ ] **Simulation parity:** Diff panel mirrors CLI schema; export button tested. +- [ ] **Workflow RBAC:** Buttons enable/disable correctly per scope (`policy:write/submit/review/approve`). +- [ ] **A11y verified:** Keyboard navigation, focus management, colour contrast (light/dark) pass automated Axe checks. +- [ ] **Offline safeguards:** Sealed-mode banner and bundle export flows present; no network calls trigger in sealed mode. +- [ ] **Telemetry wired:** Action spans and error logs include policyId, version, traceId. +- [ ] **Docs cross-links:** Links to DSL, lifecycle, runs, API, CLI guides validated. +- [ ] **Screenshot placeholders updated:** Replace TODO images with latest UI captures before GA. + +--- + +*Last updated: 2025-10-26 (Sprint 20).* + diff --git a/docs/ui/runs.md b/docs/ui/runs.md new file mode 100644 index 00000000..1e7815cd --- /dev/null +++ b/docs/ui/runs.md @@ -0,0 +1,169 @@ +# StellaOps Console - Runs Workspace + +> **Audience:** Scheduler Guild, Console UX, operators, support engineers. +> **Scope:** Runs dashboard, live progress, queue management, diffs, retries, evidence downloads, observability, troubleshooting, and offline behaviour (Sprint 23). + +The Runs workspace surfaces Scheduler activity across tenants: upcoming schedules, active runs, progress, deltas, and evidence bundles. It helps operators monitor backlog, drill into run segments, and recover from failures without leaving the console. + +--- + +## 1. Access and prerequisites + +- **Route:** `/console/runs` (list) with detail drawer `/console/runs/:runId`. SSE stream at `/console/runs/:runId/stream`. +- **Scopes:** `runs.read` (baseline), `runs.manage` (cancel/retry), `policy:runs` (view policy deltas), `downloads.read` (evidence bundles). +- **Dependencies:** Scheduler WebService (`/runs`, `/schedules`, `/preview`), Scheduler Worker event feeds, Policy Engine run summaries, Scanner WebService evidence endpoints. +- **Feature flags:** `runs.dashboard.enabled`, `runs.sse.enabled`, `runs.retry.enabled`, `runs.evidenceBundles`. +- **Tenancy:** Tenant selector filters list; cross-tenant admins can pin multiple tenants side-by-side (split view). + +--- + +## 2. Layout overview + +``` ++-------------------------------------------------------------------+ +| Header: Tenant badge - schedule selector - backlog metrics | ++-------------------------------------------------------------------+ +| Cards: Active runs - Queue depth - New findings - KEV deltas | ++-------------------------------------------------------------------+ +| Tabs: Active | Completed | Scheduled | Failures | ++-------------------------------------------------------------------+ +| Runs table (virtualised) | +| Columns: Run ID | Trigger | State | Progress | Duration | Deltas | ++-------------------------------------------------------------------+ +| Detail drawer: Summary | Segments | Deltas | Evidence | Logs | ++-------------------------------------------------------------------+ +``` + +The header integrates the status ticker to show ingestion deltas and planner heartbeat. + +--- + +## 3. Runs table + +| Column | Description | +|--------|-------------| +| **Run ID** | Deterministic identifier (`run:::`). Clicking opens detail drawer. | +| **Trigger** | `cron`, `manual`, `feedser`, `vexer`, `policy`, `content-refresh`. Tooltip lists schedule and initiator. | +| **State** | Badges: `planning`, `queued`, `running`, `completed`, `cancelled`, `error`. Errors include error code (e.g., `ERR_RUN_005`). | +| **Progress** | Percentage + processed/total candidates. SSE updates increment in real time. | +| **Duration** | Elapsed time (auto-updating). Completed runs show total duration; running runs show timer. | +| **Deltas** | Count of findings deltas (`+critical`, `+high`, `-quieted`, etc.). Tooltip expands severity breakdown. | + +Row badges include `KEV first`, `Content refresh`, `Policy promotion follow-up`, and `Retry`. Selecting multiple rows enables bulk downloads and exports. + +Filters: trigger type, state, schedule, severity impact (critical/high), policy revision, timeframe, planner shard, error code. + +--- + +## 4. Detail drawer + +Sections: + +1. **Summary** - run metadata (tenant, trigger, linked schedule, planner shard count, started/finished timestamps, correlation ID). +2. **Progress** - segmented progress bar (planner, queue, execution, post-processing). Real-time updates via SSE; includes throughput (targets per minute). +3. **Segments** - table of run segments with state, target count, executor, retry count. Operators can retry failed segments individually (requires `runs.manage`). +4. **Deltas** - summary of findings changes (new findings, resolved findings, severity shifts, KEV additions). Links to Findings view filtered by run ID. +5. **Evidence** - links to evidence bundles (JSON manifest, DSSE attestation), policy run records, and explain bundles. Download buttons use `/console/exports` orchestration. +6. **Logs** - last 50 structured log entries with severity, message, correlation ID; scroll-to-live for streaming logs. `Open in logs` copies query for external log tooling. + +--- + +## 5. Queue and schedule management + +- Schedule side panel lists upcoming jobs with cron expressions, time zones, and enable toggles. +- Queue depth chart shows current backlog per tenant and schedule (planner backlog, executor backlog). +- "Preview impact" button opens modal for manual run planning (purls or vuln IDs) and shows impacted image count before launch. CLI parity: `stella runs preview --tenant --file keys.json`. +- Manual run form allows selecting mode (`analysis-only`, `content-refresh`), scope, and optional policy snapshot. +- Pausing a schedule requires confirmation; UI displays earliest next run after resume. + +--- + +## 6. Live updates and SSE stream + +- SSE endpoint `/console/runs/{id}/stream` streams JSON events (`stateChanged`, `segmentProgress`, `deltaSummary`, `log`). UI reconnects with exponential backoff and heartbeat. +- Global ticker shows planner heartbeat age; banner warns after 90 seconds of silence. +- Offline mode disables SSE and falls back to polling every 30 seconds. + +--- + +## 7. Retry and remediation + +- Failed segments show retry button; UI displays reason and cooldown timers. Retry actions are scope-gated and logged. +- Full run retry resets segments while preserving original run metadata; new run ID references previous run in `retryOf` field. +- "Escalate to support" button opens incident template pre-filled with run context and correlation IDs. +- Troubleshooting quick links: + - `ERR_RUN_001` (planner lock) + - `ERR_RUN_005` (Scanner timeout) + - `ERR_RUN_009` (impact index stale) + Each link points to corresponding runbook sections (`docs/ops/scheduler-runbook.md`). +- CLI parity: `stella runs retry --run `, `stella runs cancel --run `. + +--- + +## 8. Evidence downloads + +- Evidence tab aggregates: + - Policy run summary (`/policy/runs/{id}`) + - Findings delta CSV (`/downloads/findings/{runId}.csv`) + - Scanner evidence bundle (compressed JSON with manifest) +- Downloads show size, hash, signature status. +- "Bundle for offline" packages all evidence into single tarball with manifest/digest; UI notes CLI parity (`stella runs export --run --bundle`). +- Completed bundles stored in Downloads workspace for reuse (links provided). + +--- + +## 9. Observability + +- Metrics cards: `scheduler_queue_depth`, `scheduler_runs_active`, `scheduler_runs_error_total`, `scheduler_runs_duration_seconds`. +- Trend charts: queue depth (last 24h), runs per trigger, average duration, determinism score. +- Alert banners: planner lag > SLA, queue depth > threshold, repeated error codes. +- Telemetry panel lists latest events (e.g., `scheduler.run.started`, `scheduler.run.completed`, `scheduler.run.failed`). + +--- + +## 10. Offline and air-gap behaviour + +- Offline banner highlights snapshot timestamp and indicates SSE disabled. +- Manual run form switches to generate CLI script for offline execution (`stella runs submit --bundle `). +- Evidence download buttons output local paths; UI reminds to copy to removable media. +- Queue charts use snapshot data; manual refresh button loads latest records from Offline Kit. +- Tenants absent from snapshot hidden to avoid partial data. + +--- + +## 11. Screenshot coordination + +- Placeholders: + - `![Runs dashboard placeholder](../assets/ui/runs/dashboard-placeholder.png)` + - `![Run detail placeholder](../assets/ui/runs/detail-placeholder.png)` +- Coordinate with Scheduler Guild for updated screenshots after Sprint 23 UI stabilises (tracked in `#console-screenshots`, entry 2025-10-26). + +--- + +## 12. References + +- `/docs/ui/console-overview.md` - shell, SSE ticker. +- `/docs/ui/navigation.md` - route map and deep links. +- `/docs/ui/findings.md` - findings filtered by run. +- `/docs/ui/downloads.md` - download manager, export retention, CLI parity. +- `/docs/ARCHITECTURE_SCHEDULER.md` - scheduler architecture and data model. +- `/docs/policy/runs.md` - policy run integration. +- `/docs/cli/policy.md` and `/docs/cli/policy.md` section 5 for CLI parity (runs commands pending). +- `/docs/ops/scheduler-runbook.md` - troubleshooting. + +--- + +## 13. Compliance checklist + +- [ ] Runs table columns, filters, and states described. +- [ ] Detail drawer sections documented (segments, deltas, evidence, logs). +- [ ] Queue management, manual run, and preview coverage included. +- [ ] SSE and live update behaviour detailed. +- [ ] Retry, remediation, and runbook references provided. +- [ ] Evidence downloads and bundle workflows documented with CLI parity. +- [ ] Offline behaviour and screenshot coordination recorded. +- [ ] References validated. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* diff --git a/docs/ui/sbom-explorer.md b/docs/ui/sbom-explorer.md new file mode 100644 index 00000000..87d16d78 --- /dev/null +++ b/docs/ui/sbom-explorer.md @@ -0,0 +1,195 @@ +# StellaOps Console - SBOM Explorer + +> **Audience:** Console UX, SBOM Service Guild, enablement teams, customer onboarding. +> **Scope:** Catalog listing, component detail, graph overlays, exports, performance hints, and offline behaviour for the SBOM Explorer that ships in Sprint 23. + +The SBOM Explorer lets operators inspect software bills of materials collected by Scanner and normalised by the SBOM Service. It provides tenant-scoped catalogs, usage overlays, provenance-aware graphs, and deterministic export paths that align with CLI workflows. + +--- + +## 1. Access and prerequisites + +- **Routes:** `/console/sbom` (catalog) and `/console/sbom/:digest` (detail). +- **Scopes:** `sbom.read` (required), `sbom.export` for large export jobs, `findings.read` to open explain drawers, `policy.read` to view overlay metadata. +- **Feature flags:** `sbomExplorer.enabled` (default true when SBOM Service v3 API is enabled) and `graph.overlays.enabled` for Cartographer-backed overlays. +- **Tenant scoping:** All queries include `tenant` tokens; switching tenants triggers catalog refetch and clears cached overlays. +- **Data dependencies:** Requires SBOM Service 3.1+ with Cartographer overlays and Policy Engine explain hints enabled. + +--- + +## 2. Layout overview + +``` ++-----------------------------------------------------------------------+ +| Header: Tenant badge - global filters - offline indicator - actions | ++-----------------------------------------------------------------------+ +| Left rail: Saved views - pinned tags - export queue status | ++-----------------------------------------------------------------------+ +| Catalog table (virtualised) | +| - Columns: Image digest - Source - Scan timestamp - Policy verdict | +| - Badges: Delta SBOM, Attested, Offline snapshot | ++-----------------------------------------------------------------------+ +| Detail drawer or full page tabs (Inventory | Usage | Components | | +| Overlays | Explain | Exports) | ++-----------------------------------------------------------------------+ +``` + +The catalog and detail views reuse the shared command palette, context chips, and SSE status ticker described in `/docs/ui/navigation.md`. + +--- + +## 3. Catalog view + +| Feature | Description | +|---------|-------------| +| **Virtual table** | Uses Angular CDK virtual scroll to render up to 10,000 records per tenant without layout jank. Sorting and filtering are client-side for <= 20k rows; the UI upgrades to server-side queries automatically when more records exist. | +| **Preset segments** | Quick toggles for `All`, `Recent (7 d)`, `Delta-ready`, `Attested`, and `Offline snapshots`. Each preset maps to saved view tokens for CLI parity. | +| **Search** | Global search field supports image digests, repository tags, SBOM IDs, and component PURLs. Search terms propagate to the detail view when opened. | +| **Badges** | - `Delta` badge indicates SBOM produced via delta mode (layers reuse).
- `Attested` badge links to Attestor proof and Rekor record.
- `Snapshot` badge shows offline import hash.
- `Policy` badge references last policy verdict summary. | +| **Bulk actions** | Multi-select rows to stage export jobs, trigger async explain generation, or copy CLI commands. Actions enforce per-tenant rate limits and show authority scopes in tooltips. | + +--- + +## 4. Detail tabs + +### 4.1 Inventory tab + +- Default view summarising all components with columns for package name (PURL), version, supplier, license, size, and counts of referencing layers. +- Filters: severity, ecosystem (OS, NPM, PyPI, Maven, Go, NuGet, Rust, containers), usage flag (true/false), package tags. +- Sorting: by severity (desc), version (asc), supplier. +- Cell tooltips reference Concelier advisories and Policy Engine findings when available. +- Total component count, unique suppliers, and critical severity counts appear in the header cards. + +### 4.2 Usage tab + +- Focuses on runtime usage (EntryTrace, runtime sensors, allow lists). +- Columns include process names, entry points, and `usedByEntrypoint` flags. +- Grouping: by entry point, by package, or by namespace (Kubernetes). +- Highlights mismatches between declared dependencies and observed usage for drift detection. + +### 4.3 Components tab + +- Deep dive for a single component selected from Inventory or Usage. +- Shows provenance timeline (introduced in layer, modified, removed), file paths, cryptographic hashes, and linked evidence (DSSE, Attestor bundles). +- Links to CLI commands: `stella sbom component show ` and `stella sbom component export`. +- Drawer supports multi-component comparison through tabbed interface. + +### 4.4 Overlays tab + +- Displays Cartographer overlays: vulnerability overlays (policy verdicts), runtime overlays (process traces), and vendor advisories. +- Each overlay card lists source, generation timestamp, precedence, and staleness relative to tenant SLA. +- Toggle overlays on/off to see impact on component status; UI does not mutate canonical SBOM, it only enriches the view. +- Graph preview button opens force-directed component graph (limited to <= 500 nodes) with filters for dependency depth and relationship type. +- Overlay metadata includes the CLI parity snippet: `stella sbom overlay apply --overlay --digest `. + +### 4.5 Explain tab + +- Integrates Policy Engine explain drawer. +- Shows rule hits, VEX overrides, and evidence per component. +- Provides "Open in Findings" link that preserves tenant and filters. + +### 4.6 Exports tab + +- Lists available exports (CycloneDX JSON, CycloneDX Protobuf, SPDX JSON, SPDX Tag-Value, Delta bundle, Evidence bundle). +- Each export entry shows size, hash (SHA-256), format version, and generation time. +- Download buttons respect RBAC and offline quotas; CLI callouts mirror `stella sbom export`. +- "Schedule export" launches async job for large bundles; job status integrates with `/console/downloads`. +- Includes copy-to-clipboard path for offline transfers (`/offline-kits/export///`). + +--- + +## 5. Filters and presets + +| Filter | Applies to | Notes | +|--------|------------|-------| +| **Severity** | Inventory, Overlays, Explain | Uses Policy Engine severity buckets and KEV flag. | +| **Ecosystem** | Inventory, Usage | Multi-select list with search; maps to package type derived from PURL. | +| **License** | Inventory | Groups by SPDX identifiers; warns on copyleft obligations. | +| **Supplier** | Inventory, Components | Autocomplete backed by SBOM metadata. | +| **Tags** | Inventory, Usage | Tags provided by Scanner or user-defined metadata. | +| **Component search** | Components, Overlays | Accepts PURL or substring; retains highlight when switching tabs. | +| **Snapshot** | Catalog | Filters to SBOMs sourced from Offline Kit or local import. | +| **Attested only** | Catalog, Exports | Limits to SBOMs signed by Attestor; displays Rekor badge. | + +Saved views store combinations of these filters and expose command palette shortcuts (`Cmd+1-9 / Ctrl+1-9`). + +--- + +## 6. Graph overlays and cartography + +- Graph view is powered by Cartographer projections (tenant-scoped graph snapshots). +- Supported overlays: + - **Dependency graph** (default) - nodes represent components, edges represent dependencies with direction (introducer -> introduced). + - **Runtime call graph** - optional overlay layering process calls on top of dependencies. + - **Vulnerability overlay** - colours nodes by highest severity and outlines exploited components. +- Controls: depth slider (1-6), include transitive flag, hide dev dependencies toggle, highlight vendor-specified critical paths. +- Export options: GraphML, JSON Lines, and screenshot capture (requires `graph.export`). +- Performance guardrails: overlays warn when node count exceeds 2,000; user can queue background job to render static graph for download instead. + +--- + +## 7. Exports and automation + +- **Instant exports:** Inline downloads for CycloneDX JSON/Protobuf (<= 25 MB) and SPDX JSON (<= 25 MB). +- **Async exports:** Larger bundles stream through the download manager with resume support. UI polls `/console/downloads` every 15 seconds while export is in progress. +- **CLI parity:** Each export card displays the equivalent CLI command and environment variables (proxy, offline). +- **Compliance metadata:** Export manifests include SBOM ID, component count, hash, signature state, and policy verdict summary so auditors can validate offline. +- **Automation hooks:** Webhook button copies the `/downloads/hooks/subscribe` call for integration with CI pipelines. + +--- + +## 8. Performance tips + +- Virtual scroll keeps initial render under 70 ms for 10k rows; server-side pagination engages beyond that threshold. +- Graph overlay rendering uses Web Workers to keep main thread responsive; heavy layouts show "Background layout in progress" banner. +- SSE updates (new SBOM ready) refresh header cards and prepend rows without full table redraw. +- Prefetching: opening a detail drawer preloads overlays and exports concurrently; these requests cancel automatically if the user navigates away. +- Local cache (IndexedDB) stores last viewed SBOM detail for each tenant (up to 20 entries). Cache invalidates when new merge hash is observed. + +--- + +## 9. Offline and air-gap behaviour + +- Catalog reads from Offline Kit snapshot if gateway is in sealed mode; offline banner lists snapshot ID and staleness. +- Overlays limited to data included in snapshot; missing overlays show guidance to import updated Cartographer package. +- Exports queue locally and generate tarballs ready to copy to removable media. +- CLI parity callouts switch to offline examples (using `stella sbom export --offline`). +- Tenants unavailable in snapshot are hidden from the tenant picker to prevent inconsistent views. + +--- + +## 10. Screenshot coordination + +- Placeholder images: + - `![SBOM catalog view placeholder](../assets/ui/sbom/catalog-placeholder.png)` + - `![Overlay graph placeholder](../assets/ui/sbom/overlay-placeholder.png)` +- Coordinate with Console Guild to capture updated screenshots (dark and light theme) once Sprint 23 UI stabilises. Track follow-up in Console Guild thread `#console-screenshots` dated 2025-10-26. + +--- + +## 11. References + +- `/docs/ui/console-overview.md` - navigation shell, tenant model, filters. +- `/docs/ui/navigation.md` - command palette, deep-link schema. +- `/docs/ui/downloads.md` - download queue, manifest parity, offline export handling. +- `/docs/security/console-security.md` - scopes, DPoP, CSP. +- `/docs/cli-vs-ui-parity.md` - CLI equivalence matrix. +- `/docs/architecture/console.md` (pending) - component data flows. +- `/docs/architecture/overview.md` - high-level module relationships. +- `/docs/ingestion/aggregation-only-contract.md` - provenance and guard rails. + +--- + +## 12. Compliance checklist + +- [ ] Catalog table and detail tabs documented with columns, filters, and presets. +- [ ] Overlay behaviour describes Cartographer integration and CLI parity. +- [ ] Export section includes instant vs async workflow and compliance metadata. +- [ ] Performance considerations align with UI benchmarks (virtual scroll, workers). +- [ ] Offline behaviour captured for catalog, overlays, exports. +- [ ] Screenshot placeholders and coordination notes recorded with Console Guild follow-up. +- [ ] All referenced docs verified and accessible. + +--- + +*Last updated: 2025-10-26 (Sprint 23).* diff --git a/docs/updates/2025-10-26-authority-graph-scopes.md b/docs/updates/2025-10-26-authority-graph-scopes.md new file mode 100644 index 00000000..a3fe752e --- /dev/null +++ b/docs/updates/2025-10-26-authority-graph-scopes.md @@ -0,0 +1,15 @@ +# 2025-10-26 — Authority graph scopes documentation refresh + +## Summary + +- Documented least-privilege guidance for the new `graph:*` scopes in `docs/11_AUTHORITY.md` (scope mapping, tenant propagation, and DPoP expectations). +- Extended the sample client table/config to include Cartographer and Graph API registrations so downstream teams can copy/paste the correct defaults. +- Highlighted the requirement to consume `StellaOpsScopes` constants instead of hard-coded scope strings across services. + +## Next steps + +| Team | Follow-up | Target | +|------|-----------|--------| +| Authority Core | Ensure `/jwks` changelog references graph scope rollout in next release note. | 2025-10-28 | +| Graph API Guild | Update gateway scaffolding to request scopes from `StellaOpsScopes` once the host project lands. | Sprint 21 stand-up | +| Scheduler Guild | Confirm Cartographer client onboarding uses the new sample secret templates. | Sprint 21 stand-up | diff --git a/docs/updates/2025-10-26-scheduler-graph-jobs.md b/docs/updates/2025-10-26-scheduler-graph-jobs.md new file mode 100644 index 00000000..701955c0 --- /dev/null +++ b/docs/updates/2025-10-26-scheduler-graph-jobs.md @@ -0,0 +1,34 @@ +# 2025-10-26 — Scheduler Graph Job DTOs ready for integration + +## Summary + +SCHED-MODELS-21-001 delivered the new `GraphBuildJob`/`GraphOverlayJob` contracts and SCHED-MODELS-21-002 publishes the accompanying documentation + samples for downstream teams. + +Key links: + +- Schema doc: `src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md` +- Samples (round-trip tested): `samples/api/scheduler/graph-build-job.json`, `samples/api/scheduler/graph-overlay-job.json` +- Event schema + sample: `docs/events/scheduler.graph.job.completed@1.json`, `docs/events/samples/scheduler.graph.job.completed@1.sample.json` +- API doc: `src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-21-001-GRAPH-APIS.md` +- Tests: `StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs`, `GraphJobStateMachineTests.cs` + +## Action items + +| Guild | Request | Owner | Target | +| --- | --- | --- | --- | +| Scheduler WebService | Wire DTOs into upcoming `/graphs` job APIs (SCHED-WEB-21-001/002). | Scheduler Models Guild | Sprint 21 stand-up | +| Scheduler Worker | Align planners/executors with `GraphJobStateMachine` and new metadata fields. | Scheduler Models Guild | Sprint 21 stand-up | +| Cartographer | Confirm expectations for `graphSnapshotId`, `cartographerJobId`, overlay triggers. | Scheduler Models Guild | Cartographer sync 2025-10-27 | + +### Notification log + +- 2025-10-26 — Posted summary + action items to `#scheduler-guild` and `#cartographer-guild` using the snippet below. Both messages linked back to the schema doc and event sample for follow-up. +- 2025-10-26 — Shared the API doc link with WebService guild thread for endpoint contract review before Cartographer wiring. Highlighted new `POST /graphs/hooks/completed` + `GET /graphs/overlays/lag` behaviour and correlation IDs. + +> Suggested message for Slack `#scheduler-guild` & `#cartographer-guild`: +> +> ``` +> Graph job DTOs/docs are live (SCHED-MODELS-21-001/002). Samples under samples/api/scheduler, schema notes in src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md. Please review before wiring SCHED-WEB-21-001/201. GraphJobStateMachine enforces status/attempt invariants—shout if you need additional states. +> ``` + +Record notifications here once posted. diff --git a/etc/authority.yaml b/etc/authority.yaml new file mode 100644 index 00000000..5cf1b5d8 --- /dev/null +++ b/etc/authority.yaml @@ -0,0 +1,151 @@ +# StellaOps Authority configuration (dev profile) +# Derived from etc/authority.yaml.sample; trimmed to the services needed for local +# stacks and kept under version control so compose/helm bundles mount a working config. + +schemaVersion: 1 + +issuer: "https://authority.localtest.me" + +accessTokenLifetime: "00:15:00" +refreshTokenLifetime: "30.00:00:00" +identityTokenLifetime: "00:05:00" +authorizationCodeLifetime: "00:05:00" +deviceCodeLifetime: "00:15:00" + +storage: + connectionString: "mongodb://stellaops:stellaops@mongo:27017/stellaops_authority" + databaseName: "stellaops_authority" + commandTimeout: "00:00:30" + +signing: + enabled: true + activeKeyId: "authority-signing-dev" + keyPath: "../certificates/authority-signing-dev.pem" + algorithm: "ES256" + keySource: "file" + +bootstrap: + enabled: false + apiKey: "change-me" + defaultIdentityProvider: "standard" + +pluginDirectories: + - "../StellaOps.Authority.PluginBinaries" + +plugins: + configurationDirectory: "../etc/authority.plugins" + descriptors: + standard: + type: "standard" + assemblyName: "StellaOps.Authority.Plugin.Standard" + enabled: true + configFile: "standard.yaml" + capabilities: + - password + - bootstrap + - clientProvisioning + metadata: + defaultRole: "operators" + +clients: + - clientId: "policy-engine" + displayName: "Policy Engine Service" + grantTypes: [ "client_credentials" ] + audiences: [ "api://policy-engine" ] + scopes: [ "policy:run", "findings:read", "effective:write" ] + tenant: "tenant-default" + properties: + serviceIdentity: "policy-engine" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/policy-engine.secret" + + - clientId: "cartographer-service" + displayName: "Cartographer Service" + grantTypes: [ "client_credentials" ] + audiences: [ "api://cartographer" ] + scopes: [ "graph:write", "graph:read" ] + tenant: "tenant-default" + properties: + serviceIdentity: "cartographer" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/cartographer-service.secret" + + - clientId: "graph-api" + displayName: "Graph API Gateway" + grantTypes: [ "client_credentials" ] + audiences: [ "api://graph-api" ] + scopes: [ "graph:read", "graph:export", "graph:simulate" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/graph-api.secret" + + - clientId: "concelier-ingest" + displayName: "Concelier Ingestion" + grantTypes: [ "client_credentials" ] + audiences: [ "api://concelier" ] + scopes: [ "advisory:ingest", "advisory:read" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/concelier-ingest.secret" + + - clientId: "excitor-ingest" + displayName: "Excititor VEX Ingestion" + grantTypes: [ "client_credentials" ] + audiences: [ "api://excitor" ] + scopes: [ "vex:ingest", "vex:read" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/excitor-ingest.secret" + + - clientId: "graph-api-cli" + displayName: "Graph Explorer CLI" + grantTypes: [ "client_credentials" ] + audiences: [ "api://graph-api" ] + scopes: [ "graph:read", "graph:export" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/graph-api-cli.secret" + +security: + rateLimiting: + token: + enabled: true + permitLimit: 30 + window: "00:01:00" + queueLimit: 0 + authorize: + enabled: true + permitLimit: 60 + window: "00:01:00" + queueLimit: 10 + passwordHashing: + algorithm: "Argon2id" + memorySizeInKib: 19456 + iterations: 2 + parallelism: 1 + senderConstraints: + dpop: + enabled: true + proofLifetime: "00:05:00" + allowedClockSkew: "00:00:10" + replayWindow: "00:10:00" + nonce: + enabled: false + mtls: + enabled: false + +bypassNetworks: + - "127.0.0.1/32" + - "::1/128" diff --git a/etc/authority.yaml.sample b/etc/authority.yaml.sample index 3f3aec62..4657f63d 100644 --- a/etc/authority.yaml.sample +++ b/etc/authority.yaml.sample @@ -1,113 +1,197 @@ -# StellaOps Authority configuration template. -# Copy to ../etc/authority.yaml (relative to the Authority content root) -# and adjust values to fit your environment. Environment variables -# prefixed with STELLAOPS_AUTHORITY_ override these values at runtime. -# Example: STELLAOPS_AUTHORITY__ISSUER=https://authority.example.com - -schemaVersion: 1 - -# Absolute issuer URI advertised to clients. Use HTTPS for anything -# beyond loopback development. -issuer: "https://authority.stella-ops.local" - -# Token lifetimes expressed as HH:MM:SS or DD.HH:MM:SS. -accessTokenLifetime: "00:15:00" -refreshTokenLifetime: "30.00:00:00" -identityTokenLifetime: "00:05:00" -authorizationCodeLifetime: "00:05:00" -deviceCodeLifetime: "00:15:00" - -# MongoDB storage connection details. -storage: - connectionString: "mongodb://localhost:27017/stellaops-authority" - # databaseName: "stellaops_authority" - commandTimeout: "00:00:30" - -# Signing configuration for revocation bundles and JWKS. -signing: - enabled: true - activeKeyId: "authority-signing-2025-dev" - keyPath: "../certificates/authority-signing-2025-dev.pem" - algorithm: "ES256" - keySource: "file" - # provider: "default" - additionalKeys: - - keyId: "authority-signing-dev" - path: "../certificates/authority-signing-dev.pem" - source: "file" - # Rotation flow: - # 1. Generate a new PEM under ./certificates (e.g. authority-signing-2026-dev.pem). - # 2. Trigger the .gitea/workflows/authority-key-rotation.yml workflow (or run - # ops/authority/key-rotation.sh) with the new keyId/keyPath. - # 3. Update activeKeyId/keyPath above and move the previous key into additionalKeys - # so restarts retain retired material for JWKS consumers. - -# Bootstrap administrative endpoints (initial provisioning). -bootstrap: - enabled: false - apiKey: "change-me" - defaultIdentityProvider: "standard" - -# Directories scanned for Authority plug-ins. Relative paths resolve -# against the application content root, enabling air-gapped deployments -# that package plug-ins alongside binaries. -pluginDirectories: - - "../StellaOps.Authority.PluginBinaries" - # "/var/lib/stellaops/authority/plugins" - -# Plug-in manifests live in descriptors below; per-plugin settings are stored -# in the configurationDirectory (YAML files). Authority will load any enabled -# plugins and surface their metadata/capabilities to the host. -plugins: - configurationDirectory: "../etc/authority.plugins" - descriptors: - standard: - type: "standard" - assemblyName: "StellaOps.Authority.Plugin.Standard" - enabled: true - configFile: "standard.yaml" - capabilities: - - password - - bootstrap - - clientProvisioning - metadata: - defaultRole: "operators" - # Example for an external identity provider plugin. Leave disabled unless - # the plug-in package exists under StellaOps.Authority.PluginBinaries. - ldap: - type: "ldap" - assemblyName: "StellaOps.Authority.Plugin.Ldap" - enabled: false - configFile: "ldap.yaml" - capabilities: - - password - - mfa - -# OAuth client registrations issued by Authority. These examples cover Notify WebService -# in dev (notify.dev audience) and production (notify audience). Replace the secret files -# with paths to your sealed credentials before enabling bootstrap mode. -clients: - - clientId: "notify-web-dev" - displayName: "Notify WebService (dev)" - grantTypes: [ "client_credentials" ] - audiences: [ "notify.dev" ] - scopes: [ "notify.read", "notify.admin" ] - senderConstraint: "dpop" - auth: - type: "client_secret" - secretFile: "../secrets/notify-web-dev.secret" - - clientId: "notify-web" - displayName: "Notify WebService" - grantTypes: [ "client_credentials" ] - audiences: [ "notify" ] - scopes: [ "notify.read", "notify.admin" ] - senderConstraint: "dpop" - auth: - type: "client_secret" - secretFile: "../secrets/notify-web.secret" - -# CIDR ranges that bypass network-sensitive policies (e.g. on-host cron jobs). -# Keep the list tight: localhost is sufficient for most air-gapped installs. +# StellaOps Authority configuration template. +# Copy to ../etc/authority.yaml (relative to the Authority content root) +# and adjust values to fit your environment. Environment variables +# prefixed with STELLAOPS_AUTHORITY_ override these values at runtime. +# Example: STELLAOPS_AUTHORITY__ISSUER=https://authority.example.com + +schemaVersion: 1 + +# Absolute issuer URI advertised to clients. Use HTTPS for anything +# beyond loopback development. +issuer: "https://authority.stella-ops.local" + +# Token lifetimes expressed as HH:MM:SS or DD.HH:MM:SS. +accessTokenLifetime: "00:15:00" +refreshTokenLifetime: "30.00:00:00" +identityTokenLifetime: "00:05:00" +authorizationCodeLifetime: "00:05:00" +deviceCodeLifetime: "00:15:00" + +# MongoDB storage connection details. +storage: + connectionString: "mongodb://localhost:27017/stellaops-authority" + # databaseName: "stellaops_authority" + commandTimeout: "00:00:30" + +# Signing configuration for revocation bundles and JWKS. +signing: + enabled: true + activeKeyId: "authority-signing-2025-dev" + keyPath: "../certificates/authority-signing-2025-dev.pem" + algorithm: "ES256" + keySource: "file" + # provider: "default" + additionalKeys: + - keyId: "authority-signing-dev" + path: "../certificates/authority-signing-dev.pem" + source: "file" + # Rotation flow: + # 1. Generate a new PEM under ./certificates (e.g. authority-signing-2026-dev.pem). + # 2. Trigger the .gitea/workflows/authority-key-rotation.yml workflow (or run + # ops/authority/key-rotation.sh) with the new keyId/keyPath. + # 3. Update activeKeyId/keyPath above and move the previous key into additionalKeys + # so restarts retain retired material for JWKS consumers. + +# Bootstrap administrative endpoints (initial provisioning). +bootstrap: + enabled: false + apiKey: "change-me" + defaultIdentityProvider: "standard" + +# Directories scanned for Authority plug-ins. Relative paths resolve +# against the application content root, enabling air-gapped deployments +# that package plug-ins alongside binaries. +pluginDirectories: + - "../StellaOps.Authority.PluginBinaries" + # "/var/lib/stellaops/authority/plugins" + +# Plug-in manifests live in descriptors below; per-plugin settings are stored +# in the configurationDirectory (YAML files). Authority will load any enabled +# plugins and surface their metadata/capabilities to the host. +plugins: + configurationDirectory: "../etc/authority.plugins" + descriptors: + standard: + type: "standard" + assemblyName: "StellaOps.Authority.Plugin.Standard" + enabled: true + configFile: "standard.yaml" + capabilities: + - password + - bootstrap + - clientProvisioning + metadata: + defaultRole: "operators" + # Example for an external identity provider plugin. Leave disabled unless + # the plug-in package exists under StellaOps.Authority.PluginBinaries. + ldap: + type: "ldap" + assemblyName: "StellaOps.Authority.Plugin.Ldap" + enabled: false + configFile: "ldap.yaml" + capabilities: + - password + - mfa + +# OAuth client registrations issued by Authority. These examples cover Notify WebService +# in dev (notify.dev audience) and production (notify audience). Replace the secret files +# with paths to your sealed credentials before enabling bootstrap mode. +clients: + - clientId: "notify-web-dev" + displayName: "Notify WebService (dev)" + grantTypes: [ "client_credentials" ] + audiences: [ "notify.dev" ] + scopes: [ "notify.read", "notify.admin" ] + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/notify-web-dev.secret" + - clientId: "notify-web" + displayName: "Notify WebService" + grantTypes: [ "client_credentials" ] + audiences: [ "notify" ] + scopes: [ "notify.read", "notify.admin" ] + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/notify-web.secret" + - clientId: "concelier-ingest" + displayName: "Concelier Ingestion" + grantTypes: [ "client_credentials" ] + audiences: [ "api://concelier" ] + scopes: [ "advisory:ingest", "advisory:read" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/concelier-ingest.secret" + - clientId: "excitor-ingest" + displayName: "Excititor VEX Ingestion" + grantTypes: [ "client_credentials" ] + audiences: [ "api://excitor" ] + scopes: [ "vex:ingest", "vex:read" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/excitor-ingest.secret" + - clientId: "aoc-verifier" + displayName: "AOC Verification Agent" + grantTypes: [ "client_credentials" ] + audiences: [ "api://concelier", "api://excitor" ] + scopes: [ "aoc:verify" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/aoc-verifier.secret" + - clientId: "policy-engine" + displayName: "Policy Engine Service" + grantTypes: [ "client_credentials" ] + audiences: [ "api://policy-engine" ] + scopes: [ "policy:run", "findings:read", "effective:write" ] + tenant: "tenant-default" + properties: + serviceIdentity: "policy-engine" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/policy-engine.secret" + - clientId: "policy-cli" + displayName: "Policy Automation CLI" + grantTypes: [ "client_credentials" ] + audiences: [ "api://policy-engine" ] + scopes: [ "policy:write", "policy:submit", "policy:run", "findings:read" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/policy-cli.secret" + - clientId: "cartographer-service" + displayName: "Cartographer Service" + grantTypes: [ "client_credentials" ] + audiences: [ "api://cartographer" ] + scopes: [ "graph:write", "graph:read" ] + tenant: "tenant-default" + properties: + serviceIdentity: "cartographer" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/cartographer-service.secret" + - clientId: "graph-api" + displayName: "Graph API Gateway" + grantTypes: [ "client_credentials" ] + audiences: [ "api://graph-api" ] + scopes: [ "graph:read", "graph:export", "graph:simulate" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/graph-api.secret" + - clientId: "vuln-explorer-ui" + displayName: "Vuln Explorer UI" + grantTypes: [ "client_credentials" ] + audiences: [ "api://vuln-explorer" ] + scopes: [ "vuln:read" ] + tenant: "tenant-default" + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/vuln-explorer-ui.secret" + +# CIDR ranges that bypass network-sensitive policies (e.g. on-host cron jobs). +# Keep the list tight: localhost is sufficient for most air-gapped installs. bypassNetworks: - "127.0.0.1/32" - "::1/128" diff --git a/etc/concelier.yaml.sample b/etc/concelier.yaml.sample index 77c93fb5..e614e438 100644 --- a/etc/concelier.yaml.sample +++ b/etc/concelier.yaml.sample @@ -1,113 +1,119 @@ -# Concelier configuration template for StellaOps deployments. -# Copy to ../etc/concelier.yaml (relative to the web service content root) -# and adjust the values to match your environment. Environment variables -# (prefixed with CONCELIER_) override these settings at runtime. - -storage: - driver: mongo - # Mongo connection string. Use SRV URI or standard connection string. - dsn: "mongodb://concelier:concelier@mongo:27017/concelier?authSource=admin" - # Optional database name; defaults to the name embedded in the DSN or 'concelier'. - database: "concelier" - # Mongo command timeout in seconds. - commandTimeoutSeconds: 30 - -plugins: - # Concelier resolves plug-ins relative to the content root; override as needed. - baseDirectory: ".." - directory: "StellaOps.Concelier.PluginBinaries" - searchPatterns: - - "StellaOps.Concelier.Plugin.*.dll" - -telemetry: - enabled: true - enableTracing: false - enableMetrics: false - enableLogging: true - minimumLogLevel: "Information" - serviceName: "stellaops-concelier" - # Configure OTLP endpoint when shipping traces/metrics/logs out-of-band. - otlpEndpoint: "" - # Optional headers for OTLP exporters, for example authentication tokens. - otlpHeaders: {} - # Attach additional resource attributes to telemetry exports. - resourceAttributes: - deployment.environment: "local" - # Emit console exporters for local debugging. - exportConsole: true - -authority: - enabled: false - # Temporary rollout flag. When true, Concelier logs anonymous access but does not fail requests - # without tokens. Set to false before 2025-12-31 UTC to enforce authentication fully. - allowAnonymousFallback: true - # Issuer advertised by StellaOps Authority (e.g. https://authority.stella-ops.local). - issuer: "https://authority.stella-ops.local" - # Optional explicit metadata address; defaults to {issuer}/.well-known/openid-configuration. - metadataAddress: "" - requireHttpsMetadata: true - backchannelTimeoutSeconds: 30 - tokenClockSkewSeconds: 60 - audiences: - - "api://concelier" - requiredScopes: - - "concelier.jobs.trigger" - # Outbound credentials Concelier can use to call Authority (client credentials flow). - clientId: "concelier-jobs" - # Prefer storing the secret outside of the config file. Provide either clientSecret or clientSecretFile. - clientSecret: "" - clientSecretFile: "" - clientScopes: - - "concelier.jobs.trigger" - resilience: - # Enable deterministic retry/backoff when Authority is briefly unavailable. - enableRetries: true - retryDelays: - - "00:00:01" - - "00:00:02" - - "00:00:05" - # Allow stale discovery/JWKS responses when Authority is offline (extend tolerance as needed for air-gapped mirrors). - allowOfflineCacheFallback: true - offlineCacheTolerance: "00:10:00" - # Networks allowed to bypass authentication (loopback by default for on-host cron jobs). - bypassNetworks: - - "127.0.0.1/32" - - "::1/128" - -mirror: - enabled: false - # Directory containing JSON exporter outputs (absolute or relative to content root). - exportRoot: "exports/json" - # Optional explicit export identifier; defaults to `latest` symlink or most recent export. - activeExportId: "" - latestDirectoryName: "latest" - mirrorDirectoryName: "mirror" - requireAuthentication: false - maxIndexRequestsPerHour: 600 - domains: - - id: "primary" - displayName: "Primary Mirror" - requireAuthentication: false - maxDownloadRequestsPerHour: 1200 - -sources: - ghsa: - apiToken: "${GITHUB_PAT}" - pageSize: 50 - maxPagesPerFetch: 5 - requestDelay: "00:00:00.200" - failureBackoff: "00:05:00" - rateLimitWarningThreshold: 500 - secondaryRateLimitBackoff: "00:02:00" - cve: - baseEndpoint: "https://cveawg.mitre.org/api/" - apiOrg: "" - apiUser: "" - apiKey: "" - # Optional mirror used when credentials are unavailable. - seedDirectory: "./seed-data/cve" - pageSize: 200 - maxPagesPerFetch: 5 - initialBackfill: "30.00:00:00" - requestDelay: "00:00:00.250" - failureBackoff: "00:10:00" +# Concelier configuration template for StellaOps deployments. +# Copy to ../etc/concelier.yaml (relative to the web service content root) +# and adjust the values to match your environment. Environment variables +# (prefixed with CONCELIER_) override these settings at runtime. + +storage: + driver: mongo + # Mongo connection string. Use SRV URI or standard connection string. + dsn: "mongodb://concelier:concelier@mongo:27017/concelier?authSource=admin" + # Optional database name; defaults to the name embedded in the DSN or 'concelier'. + database: "concelier" + # Mongo command timeout in seconds. + commandTimeoutSeconds: 30 + +plugins: + # Concelier resolves plug-ins relative to the content root; override as needed. + baseDirectory: ".." + directory: "StellaOps.Concelier.PluginBinaries" + searchPatterns: + - "StellaOps.Concelier.Plugin.*.dll" + +telemetry: + enabled: true + enableTracing: false + enableMetrics: false + enableLogging: true + minimumLogLevel: "Information" + serviceName: "stellaops-concelier" + # Configure OTLP endpoint when shipping traces/metrics/logs out-of-band. + otlpEndpoint: "" + # Optional headers for OTLP exporters, for example authentication tokens. + otlpHeaders: {} + # Attach additional resource attributes to telemetry exports. + resourceAttributes: + deployment.environment: "local" + # Emit console exporters for local debugging. + exportConsole: true + +authority: + enabled: false + # Temporary rollout flag. When true, Concelier logs anonymous access but does not fail requests + # without tokens. Set to false before 2025-12-31 UTC to enforce authentication fully. + allowAnonymousFallback: true + # Issuer advertised by StellaOps Authority (e.g. https://authority.stella-ops.local). + issuer: "https://authority.stella-ops.local" + # Optional explicit metadata address; defaults to {issuer}/.well-known/openid-configuration. + metadataAddress: "" + requireHttpsMetadata: true + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - "api://concelier" + requiredScopes: + - "concelier.jobs.trigger" + - "advisory:read" + - "advisory:ingest" + requiredTenants: + - "tenant-default" + # Outbound credentials Concelier can use to call Authority (client credentials flow). + clientId: "concelier-jobs" + # Prefer storing the secret outside of the config file. Provide either clientSecret or clientSecretFile. + clientSecret: "" + clientSecretFile: "" + clientScopes: + - "concelier.jobs.trigger" + - "advisory:read" + - "advisory:ingest" + resilience: + # Enable deterministic retry/backoff when Authority is briefly unavailable. + enableRetries: true + retryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + # Allow stale discovery/JWKS responses when Authority is offline (extend tolerance as needed for air-gapped mirrors). + allowOfflineCacheFallback: true + offlineCacheTolerance: "00:10:00" + # Networks allowed to bypass authentication (loopback by default for on-host cron jobs). + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + +mirror: + enabled: false + # Directory containing JSON exporter outputs (absolute or relative to content root). + exportRoot: "exports/json" + # Optional explicit export identifier; defaults to `latest` symlink or most recent export. + activeExportId: "" + latestDirectoryName: "latest" + mirrorDirectoryName: "mirror" + requireAuthentication: false + maxIndexRequestsPerHour: 600 + domains: + - id: "primary" + displayName: "Primary Mirror" + requireAuthentication: false + maxDownloadRequestsPerHour: 1200 + +sources: + ghsa: + apiToken: "${GITHUB_PAT}" + pageSize: 50 + maxPagesPerFetch: 5 + requestDelay: "00:00:00.200" + failureBackoff: "00:05:00" + rateLimitWarningThreshold: 500 + secondaryRateLimitBackoff: "00:02:00" + cve: + baseEndpoint: "https://cveawg.mitre.org/api/" + apiOrg: "" + apiUser: "" + apiKey: "" + # Optional mirror used when credentials are unavailable. + seedDirectory: "./seed-data/cve" + pageSize: 200 + maxPagesPerFetch: 5 + initialBackfill: "30.00:00:00" + requestDelay: "00:00:00.250" + failureBackoff: "00:10:00" diff --git a/etc/policy-engine.yaml.sample b/etc/policy-engine.yaml.sample new file mode 100644 index 00000000..f68b8f73 --- /dev/null +++ b/etc/policy-engine.yaml.sample @@ -0,0 +1,33 @@ +# StellaOps Policy Engine configuration template. +# Copy to ../etc/policy-engine.yaml (relative to the Policy Engine content root) +# and adjust values to fit your environment. Environment variables prefixed with +# STELLAOPS_POLICY_ENGINE_ override these values at runtime. + +schemaVersion: 1 + +authority: + enabled: true + issuer: "https://authority.stella-ops.local" + clientId: "policy-engine" + clientSecret: "change-me" + scopes: [ "policy:run", "findings:read", "effective:write" ] + backchannelTimeoutSeconds: 30 + +storage: + connectionString: "mongodb://localhost:27017/policy-engine" + databaseName: "policy_engine" + commandTimeoutSeconds: 30 + +workers: + schedulerIntervalSeconds: 15 + maxConcurrentEvaluations: 4 + +resourceServer: + authority: "https://authority.stella-ops.local" + requireHttpsMetadata: true + audiences: [ "api://policy-engine" ] + requiredScopes: [ "policy:run" ] + requiredTenants: [ ] + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" diff --git a/etc/registry-signing-sample.pem b/etc/registry-signing-sample.pem new file mode 100644 index 00000000..df9e353a --- /dev/null +++ b/etc/registry-signing-sample.pem @@ -0,0 +1,27 @@ +-----BEGIN PRIVATE KEY----- +MIIEpAIBAAKCAQEApK0BkUaZC26/J7el9fnYx1Y6Uwh0b3F08r5zixK9QmuaZ0+d +Zn2m5yA/ty/G6uSVn/YU5YZd7zFTy9P7egfa/tVU5tB2Lk5/v/+6JCTG0uzQjZ1e +tfx+/j/iKnD7Z3S2CQyq4F2VQQ2xxF+SaVQ9zbmqRaWhzVtzxz6pXPVH3YYBXFjC +OXD2gG+437lPlm3CRWWPnk0hxK6SLlqVvFyP34PO8TdQF5VAcez5vFfwrkqXDBHC +vQ156P6rWTeM1g9UUfPjCmaJC9k6uM4DUUDOVOA7xemlAf+QvOIbd1Yq+XNfYo6I +WrE1bCnHgYG/Y2J17YrCX7bZ06WpDjXRHuEaiwIDAQABAoIBAQCm0sCcdwuDA1yS +g52qZ2vBEtKgeAM9H9XD7VxVMdzJx/CbCcdE289kQTZrBp3fgpovvzgzjYQeojmJ +1oU3tEJX7AD1OCXikxBvl/EunzQ7Vm25Iw2zDX/a6li3jbDHNb/P5sNhoUqS2R7z +gKqoq2oGOV3R43bHZ6N5UVoVDoRF0oZtl4Mw2aF/3JcBCQr9qsS1SoD/CPcaFc5e +2CiYtn0N+L61+Z1YblFXxUD6YJn/1XlonyDtNzW6ybMcyTj3y5DsKFnmQbF1R837 +f6LS7IZzJ9Bod3lTY2QQoW8GrYc1Y4zM0P4ZfKw7u6nLLA/A4Ngpk+kuJoha+ffq +/BQx3xU5AoGBANOltyQwXBuo5omywFbGi2+Z7UFip3yz1Vi6Yo4QzfdP/UVfhrlP +IhiIC6cvI6bf7MwwN4vShBALTbctFRZpFj5Cil1Bh/n7dJE3KKdcBFDLndCszbb5 +21N4vR7BkfGav0PpVmeRlmy3FqnpYwm0KziqFwH2tsnegegcQFzN/Q0LAoGBAMok +2PKDwYhz523kn2AVVB8pB4X0ZPrZmtHh9bPlsJS7HwioX8Oo8CD5c+WQ4u4KnYV5 +B2X2y8WTDdYH50SUSzjYCaec6Mce6CO5XrCK0pC6mvJMQBoBI35Snu461FahAE6U +zNEp2bqMx8nKnuNRw1bI8gMlMrk0dBrJRvfUsycvAoGAcOVlC5+iB9YydUzFefvK +xjBvXvG9Y60tdkN4Kd1/XiN7UjsfOCvy9EhRL1u5//JLi0O3bCtCO6fsziS0PFAO +QX9WhCok0Ifn2GwzVDfteMoqmHhPmlKL0g7G70m2JdHMIiFAMJWpbD9gWKk0o9v0 +Bk0zF0hjWG9ipN1fAv61TRUCgYEAk4fQVxbRyWYVvHHcH4scr1jYIE7so9+boQ3c +O0YDyId+rLo0Orers/5OEI2gTgLz7HzFMr2SfLWaNqMy2Beo9/C8VM5ijx2zYNvM +oN+xsZLFYoA7KM0jb5dLZ1UL84sHynwYLPy+EsB7mP+OpclAqY6cHx47CL4yxo2J +cz0KkOUCgYA4hmgwRq3di82plhMnNs14UruI9SNpZbgZ7kFCCkQbGyRhPqbwIa1U +AWhaI4SqdOskwj6B+GScXMiF49cDG4xW4Cr2/pr9F1ZcUEAWESihrINZhCg5kS2d +FSKbJ2Xqs0GGx5xAxlzUaRF8NDH6cqfynlHC9HjDJSLXquGMEmXcnw== +-----END PRIVATE KEY----- diff --git a/etc/registry-token.yaml b/etc/registry-token.yaml new file mode 100644 index 00000000..e5d1e61f --- /dev/null +++ b/etc/registry-token.yaml @@ -0,0 +1,30 @@ +registryTokenService: + authority: + issuer: "https://authority.localhost" + requireHttpsMetadata: false + audiences: + - "registry" + requiredScopes: + - "registry.token.issue" + signing: + issuer: "https://registry.localhost/token" + keyPath: "etc/registry-signing-sample.pem" + keyPassword: "" + lifetime: "00:05:00" + registry: + realm: "https://registry.localhost/v2/token" + allowedServices: + - "registry.localhost" + defaultPlan: "community" + plans: + - name: "community" + repositories: + - pattern: "stella-ops/public/*" + actions: [ "pull" ] + - name: "enterprise" + repositories: + - pattern: "stella-ops/public/*" + actions: [ "pull" ] + - pattern: "stella-ops/enterprise/*" + actions: [ "pull" ] + revokedLicenses: [] diff --git a/etc/secrets/cartographer-service.secret b/etc/secrets/cartographer-service.secret new file mode 100644 index 00000000..a222272d --- /dev/null +++ b/etc/secrets/cartographer-service.secret @@ -0,0 +1,2 @@ +# replace with a strong shared secret for the cartographer-service client +cartographer-service-secret-change-me diff --git a/etc/secrets/concelier-ingest.secret b/etc/secrets/concelier-ingest.secret new file mode 100644 index 00000000..f21fc33f --- /dev/null +++ b/etc/secrets/concelier-ingest.secret @@ -0,0 +1,2 @@ +# replace with a strong shared secret for the concelier-ingest client +concelier-ingest-secret-change-me diff --git a/etc/secrets/excitor-ingest.secret b/etc/secrets/excitor-ingest.secret new file mode 100644 index 00000000..a1fb7f12 --- /dev/null +++ b/etc/secrets/excitor-ingest.secret @@ -0,0 +1,2 @@ +# replace with a strong shared secret for the excitor-ingest client +excitor-ingest-secret-change-me diff --git a/etc/secrets/graph-api-cli.secret b/etc/secrets/graph-api-cli.secret new file mode 100644 index 00000000..9d05d27b --- /dev/null +++ b/etc/secrets/graph-api-cli.secret @@ -0,0 +1,2 @@ +# replace with a strong shared secret for the graph-api-cli client +graph-api-cli-secret-change-me diff --git a/etc/secrets/graph-api.secret b/etc/secrets/graph-api.secret new file mode 100644 index 00000000..c6f84327 --- /dev/null +++ b/etc/secrets/graph-api.secret @@ -0,0 +1,2 @@ +# replace with a strong shared secret for the graph-api client +graph-api-secret-change-me diff --git a/etc/secrets/policy-engine.secret b/etc/secrets/policy-engine.secret new file mode 100644 index 00000000..13456083 --- /dev/null +++ b/etc/secrets/policy-engine.secret @@ -0,0 +1,2 @@ +# replace with a strong shared secret for the policy-engine client +policy-engine-secret-change-me diff --git a/local-nuget/Microsoft.AspNetCore.Authentication.JwtBearer.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.AspNetCore.Authentication.JwtBearer.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..b77aa7b1 Binary files /dev/null and b/local-nuget/Microsoft.AspNetCore.Authentication.JwtBearer.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.AspNetCore.Mvc.Testing.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.AspNetCore.Mvc.Testing.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..d34a902a Binary files /dev/null and b/local-nuget/Microsoft.AspNetCore.Mvc.Testing.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.AspNetCore.OpenApi.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.AspNetCore.OpenApi.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..0d12771f Binary files /dev/null and b/local-nuget/Microsoft.AspNetCore.OpenApi.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Caching.Memory.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Caching.Memory.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..2de35f84 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Caching.Memory.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..a352044b Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.Abstractions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.Abstractions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..54cd9007 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.Abstractions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.Binder.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.Binder.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..2d625140 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.Binder.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.CommandLine.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.CommandLine.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..ff598319 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.CommandLine.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.EnvironmentVariables.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.EnvironmentVariables.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..cbde9057 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.EnvironmentVariables.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.FileExtensions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.FileExtensions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..c0524d8d Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.FileExtensions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Configuration.Json.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Configuration.Json.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..e301e844 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Configuration.Json.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.DependencyInjection.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.DependencyInjection.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..c2b42333 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.DependencyInjection.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.DependencyInjection.Abstractions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.DependencyInjection.Abstractions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..eeefd832 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.DependencyInjection.Abstractions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Diagnostics.Abstractions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Diagnostics.Abstractions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..6856db90 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Diagnostics.Abstractions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Diagnostics.HealthChecks.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Diagnostics.HealthChecks.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..21fc0ede Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Diagnostics.HealthChecks.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..e71296d1 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Hosting.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Hosting.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..738aa0fa Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Hosting.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Hosting.Abstractions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Hosting.Abstractions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..1b455945 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Hosting.Abstractions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Http.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Http.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..743d5b19 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Http.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Http.Polly.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Http.Polly.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..c2bbcfb4 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Http.Polly.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Logging.Abstractions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Logging.Abstractions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..47fae3a7 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Logging.Abstractions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Logging.Console.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Logging.Console.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..ab426b97 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Logging.Console.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Options.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Options.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..8332e807 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Options.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.Options.ConfigurationExtensions.10.0.0-rc.2.25502.107.nupkg b/local-nuget/Microsoft.Extensions.Options.ConfigurationExtensions.10.0.0-rc.2.25502.107.nupkg new file mode 100644 index 00000000..5bc4d8a3 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.Options.ConfigurationExtensions.10.0.0-rc.2.25502.107.nupkg differ diff --git a/local-nuget/Microsoft.Extensions.TimeProvider.Testing.9.10.0.nupkg b/local-nuget/Microsoft.Extensions.TimeProvider.Testing.9.10.0.nupkg new file mode 100644 index 00000000..fac63748 Binary files /dev/null and b/local-nuget/Microsoft.Extensions.TimeProvider.Testing.9.10.0.nupkg differ diff --git a/local-nuget/Microsoft.IdentityModel.Abstractions.8.14.0.nupkg b/local-nuget/Microsoft.IdentityModel.Abstractions.8.14.0.nupkg new file mode 100644 index 00000000..c52dcd82 Binary files /dev/null and b/local-nuget/Microsoft.IdentityModel.Abstractions.8.14.0.nupkg differ diff --git a/local-nuget/Microsoft.IdentityModel.Logging.8.14.0.nupkg b/local-nuget/Microsoft.IdentityModel.Logging.8.14.0.nupkg new file mode 100644 index 00000000..8e655c91 Binary files /dev/null and b/local-nuget/Microsoft.IdentityModel.Logging.8.14.0.nupkg differ diff --git a/local-nuget/Microsoft.IdentityModel.Tokens.8.14.0.nupkg b/local-nuget/Microsoft.IdentityModel.Tokens.8.14.0.nupkg new file mode 100644 index 00000000..9a117827 Binary files /dev/null and b/local-nuget/Microsoft.IdentityModel.Tokens.8.14.0.nupkg differ diff --git a/local-nuget/Microsoft.SourceLink.GitLab.8.0.0.nupkg b/local-nuget/Microsoft.SourceLink.GitLab.8.0.0.nupkg new file mode 100644 index 00000000..7089ff41 Binary files /dev/null and b/local-nuget/Microsoft.SourceLink.GitLab.8.0.0.nupkg differ diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index 85d67b30..3f1784de 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -2,5 +2,266 @@ "name": "git.stella-ops.org", "lockfileVersion": 3, "requires": true, - "packages": {} + "packages": { + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-cli": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ajv-cli/-/ajv-cli-5.0.0.tgz", + "integrity": "sha512-LY4m6dUv44HTyhV+u2z5uX4EhPYTM38Iv1jdgDJJJCyOOuqB8KtZEGjPZ2T+sh5ZIJrXUfgErYx/j3gLd3+PlQ==", + "dependencies": { + "ajv": "^8.0.0", + "fast-json-patch": "^2.0.0", + "glob": "^7.1.0", + "js-yaml": "^3.14.0", + "json-schema-migrate": "^2.0.0", + "json5": "^2.1.3", + "minimist": "^1.2.0" + }, + "bin": { + "ajv": "dist/index.js" + }, + "peerDependencies": { + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "ts-node": { + "optional": true + } + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-json-patch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-2.2.1.tgz", + "integrity": "sha512-4j5uBaTnsYAV5ebkidvxiLUYOwjQ+JSFljeqfTxCrH9bDmlCQaOJFS84oDJ2rAXZq2yskmk3ORfoP9DCwqFNig==", + "dependencies": { + "fast-deep-equal": "^2.0.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/fast-json-patch/node_modules/fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w==" + }, + "node_modules/fast-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", + "integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ] + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-migrate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/json-schema-migrate/-/json-schema-migrate-2.0.0.tgz", + "integrity": "sha512-r38SVTtojDRp4eD6WsCqiE0eNDt4v1WalBXb9cyZYw9ai5cGtBwzRNWjHzJl38w6TxFkXAIA7h+fyX3tnrAFhQ==", + "dependencies": { + "ajv": "^8.0.0" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } } diff --git a/ops/authority/README.md b/ops/authority/README.md index 8b8694d3..8784628e 100644 --- a/ops/authority/README.md +++ b/ops/authority/README.md @@ -38,6 +38,8 @@ Key environment variables (mirroring `StellaOpsAuthorityOptions`): For additional options, see `etc/authority.yaml.sample`. +> **Graph Explorer reminder:** When enabling Cartographer or Graph API components, update `etc/authority.yaml` so the `cartographer-service` client includes `properties.serviceIdentity: "cartographer"` and a tenant hint. Authority now rejects `graph:write` tokens that lack this marker, so existing deployments must apply the update before rolling out the new build. + ## Key rotation automation (OPS3) The `key-rotation.sh` helper wraps the `/internal/signing/rotate` endpoint delivered with CORE10. It can run in CI/CD once the new PEM key is staged on the Authority host volume. diff --git a/ops/deployment/TASKS.md b/ops/deployment/TASKS.md index 443baf77..92b9f0f9 100644 --- a/ops/deployment/TASKS.md +++ b/ops/deployment/TASKS.md @@ -2,7 +2,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DEVOPS-OPS-14-003 | TODO | Deployment Guild | DEVOPS-REL-14-001 | Document and script upgrade/rollback flows, channel management, and compatibility matrices per architecture. | Helm/Compose guides updated with digest pinning, automated checks committed, rollback drill recorded. | +| DEVOPS-OPS-14-003 | DONE (2025-10-26) | Deployment Guild | DEVOPS-REL-14-001 | Document and script upgrade/rollback flows, channel management, and compatibility matrices per architecture. | Helm/Compose guides updated with digest pinning, automated checks committed, rollback drill recorded. | | DOWNLOADS-CONSOLE-23-001 | TODO | Deployment Guild, DevOps Guild | DEVOPS-CONSOLE-23-002 | Maintain signed downloads manifest pipeline (images, Helm, offline bundles), publish JSON under `deploy/downloads/manifest.json`, and document sync cadence for Console + docs parity. | Pipeline generates signed manifest with checksums, automated PR updates manifest, docs updated with sync workflow, parity check in CI passes. | | DEPLOY-POLICY-27-001 | TODO | Deployment Guild, Policy Registry Guild | REGISTRY-API-27-001, DEVOPS-POLICY-27-003 | Produce Helm/Compose overlays for Policy Registry + simulation workers, including Mongo migrations, object storage buckets, signing key secrets, and tenancy defaults. | Overlays committed with deterministic digests; install docs updated; smoke deploy validated in staging. | | DEPLOY-POLICY-27-002 | TODO | Deployment Guild, Policy Guild | DEPLOY-POLICY-27-001, WEB-POLICY-27-004 | Document rollout/rollback playbooks for policy publish/promote (canary strategy, emergency freeze toggle, evidence retrieval) under `/docs/runbooks/policy-incident.md`. | Runbook published with decision tree; checklist appended; rehearsal recorded. | diff --git a/ops/devops/README.md b/ops/devops/README.md index e8b4ffa0..3bebdd38 100644 --- a/ops/devops/README.md +++ b/ops/devops/README.md @@ -17,6 +17,23 @@ by the new `.gitea/workflows/release.yml` pipeline. Outputs land under `out/release/`. Use `--no-push` to run full builds without pushing to the registry. +After the build completes, run the verifier to validate recorded hashes and artefact +presence: + +```bash +python ops/devops/release/verify_release.py --release-dir out/release +``` + +## Python analyzer smoke & signing + +`dotnet run --project tools/LanguageAnalyzerSmoke` exercises the Python language +analyzer plug-in against the golden fixtures (cold/warm timings, determinism). The +release workflow runs this harness automatically and then produces Cosign +signatures + SHA-256 sidecars for `StellaOps.Scanner.Analyzers.Lang.Python.dll` +and its `manifest.json`. Keep `COSIGN_KEY_REF`/`COSIGN_IDENTITY_TOKEN` populated so +the step can sign the artefacts; the generated `.sig`/`.sha256` files ship with the +Offline Kit bundle. + ## Required tooling - Docker 25+ with Buildx @@ -33,6 +50,10 @@ Supply signing material via environment variables: The workflow defaults to multi-arch (`linux/amd64,linux/arm64`), SBOM in CycloneDX, and SLSA provenance (`https://slsa.dev/provenance/v1`). +## Debug store extraction + +`build_release.py` now exports stripped debug artefacts for every ELF discovered in the published images. The files land under `out/release/debug/.build-id//.debug`, with metadata captured in `debug/debug-manifest.json` (and a `.sha256` sidecar). Use `jq` to inspect the manifest or `readelf -n` to spot-check a build-id. Offline Kit packaging should reuse the `debug/` directory as-is. + ## UI auth smoke (Playwright) As part of **DEVOPS-UI-13-006** the pipelines will execute the UI auth smoke @@ -51,4 +72,21 @@ ship from the public `dotnet-public` Azure DevOps feed. We mirror them into and writes packages alongside their SHA-256 checks. 3. `NuGet.config` registers the mirror (`local`), dotnet-public, and nuget.org. +Use `python3 ops/devops/validate_restore_sources.py` to prove the repo still +prefers the local mirror and that `Directory.Build.props` enforces the same order. +The validator now runs automatically in the `build-test-deploy` and `release` +workflows so CI fails fast when a feed priority regression slips in. + Detailed operator instructions live in `docs/ops/nuget-preview-bootstrap.md`. + +## Telemetry collector tooling (DEVOPS-OBS-50-001) + +- `ops/devops/telemetry/generate_dev_tls.sh` – generates a development CA and + client/server certificates for the OpenTelemetry collector overlay (mutual TLS). +- `ops/devops/telemetry/smoke_otel_collector.py` – sends OTLP traces/metrics/logs + over TLS and validates that the collector increments its receiver counters. +- `ops/devops/telemetry/package_offline_bundle.py` – re-packages collector assets for the Offline Kit. +- `deploy/compose/docker-compose.telemetry-storage.yaml` – Prometheus/Tempo/Loki stack for staging validation. + +Combine these helpers with `deploy/compose/docker-compose.telemetry.yaml` to run +a secured collector locally before rolling out the Helm-based deployment. diff --git a/ops/devops/TASKS.md b/ops/devops/TASKS.md index f814618f..965c1e8f 100644 --- a/ops/devops/TASKS.md +++ b/ops/devops/TASKS.md @@ -1,5 +1,11 @@ # DevOps Task Board +## Governance & Rules + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| DEVOPS-RULES-33-001 | DOING (2025-10-26) | DevOps Guild, Platform Leads | — | Contracts & Rules anchor:
• Gateway proxies only; Policy Engine composes overlays/simulations.
• AOC ingestion cannot merge; only lossless canonicalization.
• One graph platform: Graph Indexer + Graph API. Cartographer retired. | Rules posted in SPRINTS/TASKS; duplicates cleaned per guidance; reviewers acknowledge in changelog. | + | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | DEVOPS-HELM-09-001 | DONE | DevOps Guild | SCANNER-WEB-09-101 | Create Helm/Compose environment profiles (dev, staging, airgap) with deterministic digests. | Profiles committed under `deploy/`; docs updated; CI smoke deploy passes. | @@ -7,12 +13,16 @@ | DEVOPS-SCANNER-09-205 | DONE (2025-10-21) | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-204 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | CI job reads Redis stream during scanner smoke deploy, confirms Notify ingestion via API, alerts on failure. | | DEVOPS-PERF-10-001 | DONE | DevOps Guild | BENCH-SCANNER-10-001 | Add perf smoke job (SBOM compose <5 s target) to CI. | CI job runs sample build verifying <5 s; alerts configured. | | DEVOPS-PERF-10-002 | DONE (2025-10-23) | DevOps Guild | BENCH-SCANNER-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | CI exports JSON for dashboards; Grafana panel wired; Ops on-call doc updated with alert hook. | -| DEVOPS-AOC-19-001 | TODO | DevOps Guild, Platform Guild | WEB-AOC-19-003 | Integrate the AOC Roslyn analyzer and guard tests into CI, failing builds when ingestion projects attempt banned writes. | Analyzer runs in PR/CI pipelines, results surfaced in build summary, docs updated under `docs/ops/ci-aoc.md`. | -| DEVOPS-AOC-19-002 | TODO | DevOps Guild | CLI-AOC-19-002, CONCELIER-WEB-AOC-19-004, EXCITITOR-WEB-AOC-19-004 | Add pipeline stage executing `stella aoc verify --since` against seeded Mongo snapshots for Concelier + Excititor, publishing violation report artefacts. | Stage runs on main/nightly, fails on violations, artifacts retained, runbook documented. | -| DEVOPS-AOC-19-003 | TODO | DevOps Guild, QA Guild | CONCELIER-WEB-AOC-19-003, EXCITITOR-WEB-AOC-19-003 | Enforce unit test coverage thresholds for AOC guard suites and ensure coverage exported to dashboards. | Coverage report includes guard projects, threshold gate passes/fails as expected, dashboards refreshed with new metrics. | -| DEVOPS-OBS-50-001 | TODO | DevOps Guild, Observability Guild | TELEMETRY-OBS-50-001 | Deliver default OpenTelemetry collector deployment (Compose/Helm manifests), OTLP ingestion endpoints, and secure pipeline (authN, mTLS, tenant partitioning). Provide smoke test verifying traces/logs/metrics ingestion. | Collector manifests committed; smoke test green; docs updated; imposed rule banner reminder noted. | -| DEVOPS-OBS-50-002 | TODO | DevOps Guild, Security Guild | DEVOPS-OBS-50-001, TELEMETRY-OBS-51-002 | Stand up multi-tenant storage backends (Prometheus, Tempo/Jaeger, Loki) with retention policies, tenant isolation, and redaction guard rails. Integrate with Authority scopes for read paths. | Storage stack deployed with auth; retention configured; integration tests verify tenant isolation; runbook drafted. | -| DEVOPS-OBS-50-003 | TODO | DevOps Guild, Offline Kit Guild | DEVOPS-OBS-50-001 | Package telemetry stack configs for air-gapped installs (Offline Kit bundle, documented overrides, sample values) and automate checksum/signature generation. | Offline bundle includes collector+storage configs; checksums published; docs cross-linked; imposed rule annotation recorded. | +| DEVOPS-AOC-19-001 | BLOCKED (2025-10-26) | DevOps Guild, Platform Guild | WEB-AOC-19-003 | Integrate the AOC Roslyn analyzer and guard tests into CI, failing builds when ingestion projects attempt banned writes. | Analyzer runs in PR/CI pipelines, results surfaced in build summary, docs updated under `docs/ops/ci-aoc.md`. | +> Docs hand-off (2025-10-26): see `docs/ingestion/aggregation-only-contract.md` §5, `docs/architecture/overview.md`, and `docs/cli/cli-reference.md` for guard + verifier expectations. +| DEVOPS-AOC-19-002 | BLOCKED (2025-10-26) | DevOps Guild | CLI-AOC-19-002, CONCELIER-WEB-AOC-19-004, EXCITITOR-WEB-AOC-19-004 | Add pipeline stage executing `stella aoc verify --since` against seeded Mongo snapshots for Concelier + Excititor, publishing violation report artefacts. | Stage runs on main/nightly, fails on violations, artifacts retained, runbook documented. | +> Blocked: waiting on CLI verifier command and Concelier/Excititor guard endpoints to land (CLI-AOC-19-002, CONCELIER-WEB-AOC-19-004, EXCITITOR-WEB-AOC-19-004). +| DEVOPS-AOC-19-003 | BLOCKED (2025-10-26) | DevOps Guild, QA Guild | CONCELIER-WEB-AOC-19-003, EXCITITOR-WEB-AOC-19-003 | Enforce unit test coverage thresholds for AOC guard suites and ensure coverage exported to dashboards. | Coverage report includes guard projects, threshold gate passes/fails as expected, dashboards refreshed with new metrics. | +> Blocked: guard coverage suites and exporter hooks pending in Concelier/Excititor (CONCELIER-WEB-AOC-19-003, EXCITITOR-WEB-AOC-19-003). +| DEVOPS-OBS-50-001 | DONE (2025-10-26) | DevOps Guild, Observability Guild | TELEMETRY-OBS-50-001 | Deliver default OpenTelemetry collector deployment (Compose/Helm manifests), OTLP ingestion endpoints, and secure pipeline (authN, mTLS, tenant partitioning). Provide smoke test verifying traces/logs/metrics ingestion. | Collector manifests committed; smoke test green; docs updated; imposed rule banner reminder noted. | +| DEVOPS-OBS-50-002 | DOING (2025-10-26) | DevOps Guild, Security Guild | DEVOPS-OBS-50-001, TELEMETRY-OBS-51-002 | Stand up multi-tenant storage backends (Prometheus, Tempo/Jaeger, Loki) with retention policies, tenant isolation, and redaction guard rails. Integrate with Authority scopes for read paths. | Storage stack deployed with auth; retention configured; integration tests verify tenant isolation; runbook drafted. | +> Coordination started with Observability Guild (2025-10-26) to schedule staging rollout and provision service accounts. Staging bootstrap commands and secret names documented in `docs/ops/telemetry-storage.md`. +| DEVOPS-OBS-50-003 | DONE (2025-10-26) | DevOps Guild, Offline Kit Guild | DEVOPS-OBS-50-001 | Package telemetry stack configs for air-gapped installs (Offline Kit bundle, documented overrides, sample values) and automate checksum/signature generation. | Offline bundle includes collector+storage configs; checksums published; docs cross-linked; imposed rule annotation recorded. | | DEVOPS-OBS-51-001 | TODO | DevOps Guild, Observability Guild | WEB-OBS-51-001, DEVOPS-OBS-50-001 | Implement SLO evaluator service (burn rate calculators, webhook emitters), Grafana dashboards, and alert routing to Notifier. Provide Terraform/Helm automation. | Dashboards live; evaluator emits webhooks; alert runbook referenced; staging alert fired in test. | | DEVOPS-OBS-52-001 | TODO | DevOps Guild, Timeline Indexer Guild | TIMELINE-OBS-52-002 | Configure streaming pipeline (NATS/Redis/Kafka) with retention, partitioning, and backpressure tuning for timeline events; add CI validation of schema + rate caps. | Pipeline deployed; load test meets SLA; schema validation job passes; documentation updated. | | DEVOPS-OBS-53-001 | TODO | DevOps Guild, Evidence Locker Guild | EVID-OBS-53-001 | Provision object storage with WORM/retention options (S3 Object Lock / MinIO immutability), legal hold automation, and backup/restore scripts for evidence locker. | Storage configured with WORM; legal hold script documented; backup test performed; runbook updated. | @@ -29,35 +39,34 @@ | DEVOPS-AIRGAP-57-002 | TODO | DevOps Guild, Authority Guild | AUTH-OBS-50-001 | Configure sealed-mode CI tests that run services with sealed flag and ensure no egress occurs (iptables + mock DNS). | CI suite fails on attempted egress; reports remediation; documentation updated. | | DEVOPS-AIRGAP-58-001 | TODO | DevOps Guild, Notifications Guild | NOTIFY-AIRGAP-56-002 | Provide local SMTP/syslog container templates and health checks for sealed environments; integrate into Bootstrap Pack. | Templates deployed successfully; health checks in CI; docs updated. | | DEVOPS-AIRGAP-58-002 | TODO | DevOps Guild, Observability Guild | DEVOPS-AIRGAP-56-001, DEVOPS-OBS-51-001 | Ship sealed-mode observability stack (Prometheus/Grafana/Tempo/Loki) pre-configured with offline dashboards and no remote exporters. | Stack boots offline; dashboards available; verification script confirms zero egress. | -| DEVOPS-REL-14-001 | DOING (2025-10-23) | DevOps Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. | CI pipeline produces signed images + SBOM/attestations, manifests published with verified hashes, docs updated. | -| DEVOPS-REL-14-004 | TODO | DevOps Guild, Scanner Guild | DEVOPS-REL-14-001, SCANNER-ANALYZERS-LANG-10-309P | Extend release/offline smoke jobs to exercise the Python analyzer plug-in (warm/cold scans, determinism, signature checks). | Release/Offline pipelines run Python analyzer smoke suite; alerts hooked; docs updated with new coverage matrix. | -| DEVOPS-REL-17-002 | TODO | DevOps Guild | DEVOPS-REL-14-001, SCANNER-EMIT-17-701 | Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. | CI job writes `.debug` files under `artifacts/debug/.build-id/`, manifest + checksums published, offline kit includes cache, smoke job proves symbol lookup via build-id. | +| DEVOPS-REL-14-001 | DONE (2025-10-26) | DevOps Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. | CI pipeline produces signed images + SBOM/attestations, manifests published with verified hashes, docs updated. | +| DEVOPS-REL-14-004 | DONE (2025-10-26) | DevOps Guild, Scanner Guild | DEVOPS-REL-14-001, SCANNER-ANALYZERS-LANG-10-309P | Extend release/offline smoke jobs to exercise the Python analyzer plug-in (warm/cold scans, determinism, signature checks). | Release/Offline pipelines run Python analyzer smoke suite; alerts hooked; docs updated with new coverage matrix. | +| DEVOPS-REL-17-002 | DONE (2025-10-26) | DevOps Guild | DEVOPS-REL-14-001, SCANNER-EMIT-17-701 | Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. | CI job writes `.debug` files under `artifacts/debug/.build-id/`, manifest + checksums published, offline kit includes cache, smoke job proves symbol lookup via build-id. | +| DEVOPS-REL-17-004 | BLOCKED (2025-10-26) | DevOps Guild | DEVOPS-REL-17-002 | Ensure release workflow publishes `out/release/debug` (build-id tree + manifest) and fails when symbols are missing. | Release job emits debug artefacts, `mirror_debug_store.py` summary committed, warning cleared from build logs, docs updated. | | DEVOPS-MIRROR-08-001 | DONE (2025-10-19) | DevOps Guild | DEVOPS-REL-14-001 | Stand up managed mirror profiles for `*.stella-ops.org` (Concelier/Excititor), including Helm/Compose overlays, multi-tenant secrets, CDN caching, and sync documentation. | Infra overlays committed, CI smoke deploy hits mirror endpoints, runbooks published for downstream sync and quota management. | -| DEVOPS-SEC-10-301 | DONE (2025-10-20) | DevOps Guild | Wave 0A complete | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs. | Dependencies bumped to patched releases, audit logs free of NU1902/NU1903 warnings, regression tests green, change log documents upgrade guidance. | -| DEVOPS-CONSOLE-23-001 | TODO | DevOps Guild, Console Guild | CONSOLE-CORE-23-001 | Add console CI workflow (pnpm cache, lint, type-check, unit, Storybook a11y, Playwright, Lighthouse) with offline runners and artifact retention for screenshots/reports. | Workflow runs on PR & main, caches reduce install time, failing checks block merges, artifacts uploaded for triage, docs updated. | +> Note (2025-10-26, BLOCKED): IdentityModel.Tokens patched for logging 9.x, but release bundle still fails because Docker cannot stream multi-arch build context (`unix:///var/run/docker.sock` unavailable, EOF during copy). Retry once docker daemon/socket is healthy; until then `out/release/debug` cannot be generated. +| DEVOPS-CONSOLE-23-001 | BLOCKED (2025-10-26) | DevOps Guild, Console Guild | CONSOLE-CORE-23-001 | Add console CI workflow (pnpm cache, lint, type-check, unit, Storybook a11y, Playwright, Lighthouse) with offline runners and artifact retention for screenshots/reports. | Workflow runs on PR & main, caches reduce install time, failing checks block merges, artifacts uploaded for triage, docs updated. | +> Blocked: Console workspace and package scripts (CONSOLE-CORE-23-001..005) are not yet present; CI cannot execute pnpm/Playwright/Lighthouse until the Next.js app lands. | DEVOPS-CONSOLE-23-002 | TODO | DevOps Guild, Console Guild | DEVOPS-CONSOLE-23-001, CONSOLE-REL-23-301 | Produce `stella-console` container build + Helm chart overlays with deterministic digests, SBOM/provenance artefacts, and offline bundle packaging scripts. | Container published to registry mirror, Helm values committed, SBOM/attestations generated, offline kit job passes smoke test, docs updated. | -| DEVOPS-LAUNCH-18-100 | TODO | DevOps Guild | - | Finalise production environment footprint (clusters, secrets, network overlays) for full-platform go-live. | IaC/compose overlays committed, secrets placeholders documented, dry-run deploy succeeds in staging. | -| DEVOPS-LAUNCH-18-900 | TODO | DevOps Guild, Module Leads | Wave 0 completion | Collect “full implementation” sign-off from module owners and consolidate launch readiness checklist. | Sign-off record stored under `docs/ops/launch-readiness.md`; outstanding gaps triaged; checklist approved. | -| DEVOPS-LAUNCH-18-001 | TODO | DevOps Guild | DEVOPS-LAUNCH-18-100, DEVOPS-LAUNCH-18-900 | Production launch cutover rehearsal and runbook publication. | `docs/ops/launch-cutover.md` drafted, rehearsal executed with rollback drill, approvals captured. | +| DEVOPS-LAUNCH-18-100 | DONE (2025-10-26) | DevOps Guild | - | Finalise production environment footprint (clusters, secrets, network overlays) for full-platform go-live. | IaC/compose overlays committed, secrets placeholders documented, dry-run deploy succeeds in staging. | +| DEVOPS-LAUNCH-18-900 | DONE (2025-10-26) | DevOps Guild, Module Leads | Wave 0 completion | Collect “full implementation” sign-off from module owners and consolidate launch readiness checklist. | Sign-off record stored under `docs/ops/launch-readiness.md`; outstanding gaps triaged; checklist approved. | +| DEVOPS-LAUNCH-18-001 | DONE (2025-10-26) | DevOps Guild | DEVOPS-LAUNCH-18-100, DEVOPS-LAUNCH-18-900 | Production launch cutover rehearsal and runbook publication. | `docs/ops/launch-cutover.md` drafted, rehearsal executed with rollback drill, approvals captured. | | DEVOPS-NUGET-13-001 | DONE (2025-10-25) | DevOps Guild, Platform Leads | DEVOPS-REL-14-001 | Add .NET 10 preview feeds / local mirrors so `Microsoft.Extensions.*` 10.0 preview packages restore offline; refresh restore docs. | NuGet.config maps preview feeds (or local mirrored packages), `dotnet restore` succeeds for Excititor/Concelier solutions without ad-hoc feed edits, docs updated for offline bootstrap. | -| DEVOPS-NUGET-13-002 | TODO | DevOps Guild | DEVOPS-NUGET-13-001 | Ensure all solutions/projects prefer `local-nuget` before public sources and document restore order validation. | `NuGet.config` and solution-level configs resolve from `local-nuget` first; automated check verifies priority; docs updated for restore ordering. | -| DEVOPS-NUGET-13-003 | TODO | DevOps Guild, Platform Leads | DEVOPS-NUGET-13-002 | Sweep `Microsoft.*` NuGet dependencies pinned to 8.* and upgrade to latest .NET 10 equivalents (or .NET 9 when 10 unavailable), updating restore guidance. | Dependency audit shows no 8.* `Microsoft.*` packages remaining; CI builds green; changelog/doc sections capture upgrade rationale. | +| DEVOPS-NUGET-13-002 | DONE (2025-10-26) | DevOps Guild | DEVOPS-NUGET-13-001 | Ensure all solutions/projects prefer `local-nuget` before public sources and document restore order validation. | `NuGet.config` and solution-level configs resolve from `local-nuget` first; automated check verifies priority; docs updated for restore ordering. | +| DEVOPS-NUGET-13-003 | DONE (2025-10-26) | DevOps Guild, Platform Leads | DEVOPS-NUGET-13-002 | Sweep `Microsoft.*` NuGet dependencies pinned to 8.* and upgrade to latest .NET 10 equivalents (or .NET 9 when 10 unavailable), updating restore guidance. | Dependency audit shows no 8.* `Microsoft.*` packages remaining; CI builds green; changelog/doc sections capture upgrade rationale. | ## Policy Engine v2 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DEVOPS-POLICY-20-001 | TODO | DevOps Guild, Policy Guild | POLICY-ENGINE-20-001 | Integrate DSL linting in CI (parser/compile) to block invalid policies; add pipeline step compiling sample policies. | CI fails on syntax errors; lint logs surfaced; docs updated with pipeline instructions. | -| DEVOPS-POLICY-20-002 | TODO | DevOps Guild | DEVOPS-POLICY-20-001, POLICY-ENGINE-20-006 | Add `stella policy simulate` CI stage against golden SBOMs to detect delta explosions; publish diff artifacts. | Stage runs nightly/main; artifacts retained; alert thresholds configured. | -| DEVOPS-POLICY-20-003 | TODO | DevOps Guild, QA Guild | DEVOPS-POLICY-20-001, POLICY-ENGINE-20-005 | Determinism CI: run Policy Engine twice with identical inputs and diff outputs to guard non-determinism. | CI job compares outputs, fails on differences, logs stored; documentation updated. | +| DEVOPS-POLICY-20-001 | DONE (2025-10-26) | DevOps Guild, Policy Guild | POLICY-ENGINE-20-001 | Integrate DSL linting in CI (parser/compile) to block invalid policies; add pipeline step compiling sample policies. | CI fails on syntax errors; lint logs surfaced; docs updated with pipeline instructions. | +| DEVOPS-POLICY-20-003 | DONE (2025-10-26) | DevOps Guild, QA Guild | DEVOPS-POLICY-20-001, POLICY-ENGINE-20-005 | Determinism CI: run Policy Engine twice with identical inputs and diff outputs to guard non-determinism. | CI job compares outputs, fails on differences, logs stored; documentation updated. | +| DEVOPS-POLICY-20-004 | DOING (2025-10-26) | DevOps Guild, Scheduler Guild, CLI Guild | SCHED-MODELS-20-001, CLI-POLICY-20-002 | Automate policy schema exports: generate JSON Schema from `PolicyRun*` DTOs during CI, publish artefacts, and emit change alerts for CLI consumers (Slack + changelog). | CI stage outputs versioned schema files, uploads artefacts, notifies #policy-engine channel on change; docs/CLI references updated. | ## Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DEVOPS-GRAPH-21-001 | TODO | DevOps Guild, Cartographer Guild | CARTO-GRAPH-21-006 | Add load/perf jobs hitting graph viewport/path/diff endpoints with synthetic 50k/100k graphs; emit dashboards/alerts for SLOs. | CI perf job introduced; Grafana panels live; alerts configured for latency/SLA breaches. | -| DEVOPS-GRAPH-21-002 | TODO | DevOps Guild, UI Guild | UI-GRAPH-21-001 | Capture golden screenshots (Playwright) and JSON exports for visual regressions; wire into CI/offline kit. | Visual regression suite runs in CI; artifacts stored; failure triage docs updated. | -| DEVOPS-GRAPH-21-003 | TODO | DevOps Guild | CARTO-GRAPH-21-009, SBOM-SERVICE-21-002 | Package Cartographer + SBOM Service into offline kit bundles with seeded data/layout caches; document deployment steps. | Offline kit includes graph seeds; docs updated; smoke scripts validate airgapped startup. | ## Orchestrator Dashboard diff --git a/ops/devops/nuget-preview-packages.csv b/ops/devops/nuget-preview-packages.csv index c84c3aca..54780663 100644 --- a/ops/devops/nuget-preview-packages.csv +++ b/ops/devops/nuget-preview-packages.csv @@ -1,17 +1,30 @@ # Package,Version,SHA256,SourceBase(optional) # DotNetPublicFlat=https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Caching.Memory,10.0.0-preview.7.25380.108,8721fd1420fea6e828963c8343cd83605902b663385e8c9060098374139f9b2f,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Configuration,10.0.0-preview.7.25380.108,5a17ba4ba47f920a04ae51d80560833da82a0926d1e462af0d11c16b5da969f4,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Configuration.Binder,10.0.0-preview.7.25380.108,5a3af17729241e205fe8fbb1d458470e9603935ab2eb67cbbb06ce51265ff68f,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.DependencyInjection.Abstractions,10.0.0-preview.7.25380.108,1e9cd330d7833a3a850a7a42bbe0c729906c60bf1c359ad30a8622b50da4399b,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Hosting,10.0.0-preview.7.25380.108,3123bb019bbc0182cf7ac27f30018ca620929f8027e137bd5bdfb952037c7d29,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Hosting.Abstractions,10.0.0-preview.7.25380.108,b57625436c9eb53e3aa27445b680bb93285d0d2c91007bbc221b0c378ab016a3,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Http,10.0.0-preview.7.25380.108,daec142b7c7bd09ec1f2a86bfc3d7fe009825f5b653d310bc9e959c0a98a0f19,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Logging.Abstractions,10.0.0-preview.7.25380.108,87a495fa0b7054e134a5cf44ec8b071fe2bc3ddfb27e9aefc6375701dca2a33a,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.Options,10.0.0-preview.7.25380.108,c0657c2be3b7b894024586cf6e46a2ebc0e710db64d2645c4655b893b8487d8a,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 -Microsoft.Extensions.DependencyInjection.Abstractions,9.0.0,0a7715c24299e42b081b63b4f8e33da97b985e1de9e941b2b9e4c748b0d52fe7 -Microsoft.Extensions.Logging.Abstractions,9.0.0,8814ecf6dc2359715e111b78084ae42087282595358eb775456088f15e63eca5 -Microsoft.Extensions.Options,9.0.0,0d3e5eb80418fc8b41e4b3c8f16229e839ddd254af0513f7e6f1643970baf1c9 -Microsoft.Extensions.Options.ConfigurationExtensions,9.0.0,af5677b04552223787d942a3f8a323f3a85aafaf20ff3c9b4aaa128c44817280 -Microsoft.Data.Sqlite,9.0.0-rc.1.24451.1,770b637317e1e924f1b13587b31af0787c8c668b1d9f53f2fccae8ee8704e167 -Microsoft.AspNetCore.Authentication.JwtBearer,10.0.0-rc.1.25451.107,05f168c2db7ba79230e3fd77e84f6912bc73721c6656494df0b227867a6c2d3c,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.AspNetCore.Authentication.JwtBearer,10.0.0-rc.2.25502.107,3223f447bde9a3620477305a89520e8becafe23b481a0b423552af572439f8c2,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.AspNetCore.Mvc.Testing,10.0.0-rc.2.25502.107,b6b53c62e0abefdca30e6ca08ab8357e395177dd9f368ab3ad4bbbd07e517229,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.AspNetCore.OpenApi,10.0.0-rc.2.25502.107,f64de1fe870306053346a31263e53e29f2fdfe0eae432a3156f8d7d705c81d85,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Data.Sqlite,9.0.0-rc.1.24451.1,770b637317e1e924f1b13587b31af0787c8c668b1d9f53f2fccae8ee8704e167,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Caching.Memory,10.0.0-rc.2.25502.107,6ec6d156ed06b07cbee9fa1c0803b8d54a5f904a0bf0183172f87b63c4044426,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration,10.0.0-rc.2.25502.107,0716f72cdc99b03946c98c418c39d42208fc65f20301bd1f26a6c174646870f6,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration.Abstractions,10.0.0-rc.2.25502.107,db6e2cd37c40b5ac5ca7a4f40f5edafda2b6a8690f95a8c64b54c777a1d757c0,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration.Binder,10.0.0-rc.2.25502.107,80f04da6beef001d3c357584485c2ddc6fdbf3776cfd10f0d7b40dfe8a79ee43,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration.CommandLine,10.0.0-rc.2.25502.107,91974a95ae35bcfcd5e977427f3d0e6d3416e78678a159f5ec9e55f33a2e19af,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration.EnvironmentVariables,10.0.0-rc.2.25502.107,74d65a20e2764d5f42863f5f203b216533fc51b22fb02a8491036feb98ae5fef,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration.FileExtensions,10.0.0-rc.2.25502.107,5f97b56ea2ba3a1b252022504060351ce457f78ac9055d5fdd1311678721c1a1,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Configuration.Json,10.0.0-rc.2.25502.107,0ba362c479213eb3425f8e14d8a8495250dbaf2d5dad7c0a4ca8d3239b03c392,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.DependencyInjection,10.0.0-rc.2.25502.107,2e1b51b4fa196f0819adf69a15ad8c3432b64c3b196f2ed3d14b65136a6a8709,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.DependencyInjection.Abstractions,10.0.0-rc.2.25502.107,d6787ccf69e09428b3424974896c09fdabb8040bae06ed318212871817933352,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Diagnostics.Abstractions,10.0.0-rc.2.25502.107,b4bc47b4b4ded4ab2f134d318179537cbe16aed511bb3672553ea197929dc7d8,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Diagnostics.HealthChecks,10.0.0-rc.2.25502.107,855fd4da26b955b6b1d036390b1af10564986067b5cc6356cffa081c83eec158,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions,10.0.0-rc.2.25502.107,59f4724daed68a067a661e208f0a934f253b91ec5d52310d008e185bc2c9294c,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Hosting,10.0.0-rc.2.25502.107,ea9b1fa8e50acae720294671e6c36d4c58e20cfc9720335ab4f5ad4eba92cf62,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Hosting.Abstractions,10.0.0-rc.2.25502.107,98fa23ac82e19be221a598fc6f4b469e8b00c4ca2b7a42ad0bfea8b63bbaa9a2,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Http,10.0.0-rc.2.25502.107,c63c8bf4ca637137a561ca487b674859c2408918c4838a871bb26eb0c809a665,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Http.Polly,10.0.0-rc.2.25502.107,0b436196bcedd484796795f6a795d7a191294f1190f7a477f1a4937ef7f78110,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Logging.Abstractions,10.0.0-rc.2.25502.107,92b9a5ed62fe945ee88983af43c347429ec15691c9acb207872c548241cef961,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Logging.Console,10.0.0-rc.2.25502.107,fa1e10b5d6261675d9d2e97b9584ff9aaea2a2276eac584dfa77a1e35dcc58f5,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Options,10.0.0-rc.2.25502.107,d208acec60bec3350989694fd443e2d2f0ab583ad5f2c53a2879ade16908e5b4,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.Options.ConfigurationExtensions,10.0.0-rc.2.25502.107,c2863bb28c36fd67f308dd4af486897b512d62ecff2d96613ef954f5bef443e2,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.Extensions.TimeProvider.Testing,9.10.0,919a47156fc13f756202702cacc6e853123c84f1b696970445d89f16dfa45829,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.IdentityModel.Tokens,8.14.0,00b78c7b7023132e1d6b31d305e47524732dce6faca92dd16eb8d05a835bba7a,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 +Microsoft.SourceLink.GitLab,8.0.0,a7efb9c177888f952ea8c88bc5714fc83c64af32b70fb080a1323b8d32233973,https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/flat2 diff --git a/ops/devops/release/__pycache__/build_release.cpython-312.pyc b/ops/devops/release/__pycache__/build_release.cpython-312.pyc new file mode 100644 index 00000000..40afc31f Binary files /dev/null and b/ops/devops/release/__pycache__/build_release.cpython-312.pyc differ diff --git a/ops/devops/release/__pycache__/test_verify_release.cpython-312.pyc b/ops/devops/release/__pycache__/test_verify_release.cpython-312.pyc new file mode 100644 index 00000000..83693dfa Binary files /dev/null and b/ops/devops/release/__pycache__/test_verify_release.cpython-312.pyc differ diff --git a/ops/devops/release/__pycache__/verify_release.cpython-312.pyc b/ops/devops/release/__pycache__/verify_release.cpython-312.pyc new file mode 100644 index 00000000..246d0657 Binary files /dev/null and b/ops/devops/release/__pycache__/verify_release.cpython-312.pyc differ diff --git a/ops/devops/release/build_release.py b/ops/devops/release/build_release.py index 868944af..ad20b2b4 100644 --- a/ops/devops/release/build_release.py +++ b/ops/devops/release/build_release.py @@ -14,6 +14,7 @@ The workflow expects external tooling to be available on PATH: from __future__ import annotations import argparse +import contextlib import datetime as dt import hashlib import json @@ -21,11 +22,14 @@ import os import pathlib import re import shlex +import shutil import subprocess import sys +import tarfile import tempfile +import uuid from collections import OrderedDict -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence, Tuple REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] DEFAULT_CONFIG = REPO_ROOT / "ops/devops/release/components.json" @@ -184,6 +188,8 @@ class ReleaseBuilder: self.provenance_dir = ensure_directory(self.artifacts_dir / "provenance") self.signature_dir = ensure_directory(self.artifacts_dir / "signatures") self.metadata_dir = ensure_directory(self.artifacts_dir / "metadata") + self.debug_dir = ensure_directory(self.output_dir / "debug") + self.debug_store_dir = ensure_directory(self.debug_dir / ".build-id") self.temp_dir = pathlib.Path(tempfile.mkdtemp(prefix="stellaops-release-")) self.skip_signing = skip_signing self.tlog_upload = tlog_upload @@ -196,6 +202,9 @@ class ReleaseBuilder: "COSIGN_ALLOW_HTTP_REGISTRY": os.environ.get("COSIGN_ALLOW_HTTP_REGISTRY", "1"), "COSIGN_DOCKER_MEDIA_TYPES": os.environ.get("COSIGN_DOCKER_MEDIA_TYPES", "1"), } + # Cache resolved objcopy binaries keyed by machine identifier to avoid repeated lookups. + self._objcopy_cache: Dict[str, Optional[str]] = {} + self._missing_symbol_platforms: Dict[str, int] = {} # ---------------- # Build steps @@ -210,7 +219,8 @@ class ReleaseBuilder: components_result.append(result) helm_meta = self._package_helm() compose_meta = self._digest_compose_files() - manifest = self._compose_manifest(components_result, helm_meta, compose_meta) + debug_meta = self._collect_debug_store(components_result) + manifest = self._compose_manifest(components_result, helm_meta, compose_meta, debug_meta) return manifest def _prime_buildx_plugin(self) -> None: @@ -339,7 +349,15 @@ class ReleaseBuilder: if bundle_info: component_entry["signature"] = bundle_info if metadata_file.exists(): - component_entry["metadata"] = str(metadata_file.relative_to(self.output_dir.parent)) if metadata_file.is_relative_to(self.output_dir.parent) else str(metadata_file) + metadata_rel = ( + str(metadata_file.relative_to(self.output_dir.parent)) + if metadata_file.is_relative_to(self.output_dir.parent) + else str(metadata_file) + ) + component_entry["metadata"] = OrderedDict(( + ("path", metadata_rel), + ("sha256", compute_sha256(metadata_file)), + )) return component_entry def _sign_image(self, name: str, image_ref: str, tags: Sequence[str]) -> Optional[Mapping[str, Any]]: @@ -370,6 +388,7 @@ class ReleaseBuilder: image_ref, ]) signature_path.write_text(signature_data, encoding="utf-8") + signature_sha = compute_sha256(signature_path) signature_ref = run([ "cosign", "triangulate", @@ -380,6 +399,7 @@ class ReleaseBuilder: ( ("signature", OrderedDict(( ("path", str(signature_path.relative_to(self.output_dir.parent)) if signature_path.is_relative_to(self.output_dir.parent) else str(signature_path)), + ("sha256", signature_sha), ("ref", signature_ref), ("tlogUploaded", self.tlog_upload), ))), @@ -479,6 +499,271 @@ class ReleaseBuilder: entry["ref"] = ref return entry + def _collect_debug_store(self, components: Sequence[Mapping[str, Any]]) -> Optional[Mapping[str, Any]]: + if self.dry_run: + return None + debug_records: Dict[Tuple[str, str], OrderedDict[str, Any]] = {} + for component in components: + image_ref = component.get("image") + if not image_ref: + continue + name = component.get("name", "unknown") + entries = self._extract_debug_entries(name, image_ref) + for entry in entries: + key = (entry["platform"], entry["buildId"]) + existing = debug_records.get(key) + if existing is None: + record = OrderedDict(( + ("buildId", entry["buildId"]), + ("platform", entry["platform"]), + ("debugPath", entry["debugPath"]), + ("sha256", entry["sha256"]), + ("size", entry["size"]), + ("components", [entry["component"]]), + ("images", [entry["image"]]), + ("sources", list(entry["sources"])), + )) + debug_records[key] = record + else: + if entry["sha256"] != existing["sha256"]: + raise RuntimeError( + f"Build-id {entry['buildId']} for platform {entry['platform']} produced conflicting hashes" + ) + if entry["component"] not in existing["components"]: + existing["components"].append(entry["component"]) + if entry["image"] not in existing["images"]: + existing["images"].append(entry["image"]) + for source in entry["sources"]: + if source not in existing["sources"]: + existing["sources"].append(source) + if not debug_records: + sys.stderr.write( + "[error] release build produced no debug artefacts; enable symbol extraction so out/release/debug is populated (DEVOPS-REL-17-004).\n" + ) + # Remove empty directories before failing + with contextlib.suppress(FileNotFoundError, OSError): + if not any(self.debug_store_dir.iterdir()): + self.debug_store_dir.rmdir() + with contextlib.suppress(FileNotFoundError, OSError): + if not any(self.debug_dir.iterdir()): + self.debug_dir.rmdir() + raise RuntimeError( + "Debug store collection produced no build-id artefacts (DEVOPS-REL-17-004)." + ) + entries = [] + for record in debug_records.values(): + entry = OrderedDict(( + ("buildId", record["buildId"]), + ("platform", record["platform"]), + ("debugPath", record["debugPath"]), + ("sha256", record["sha256"]), + ("size", record["size"]), + ("components", sorted(record["components"])), + ("images", sorted(record["images"])), + ("sources", sorted(record["sources"])), + )) + entries.append(entry) + entries.sort(key=lambda item: (item["platform"], item["buildId"])) + manifest_path = self.debug_dir / "debug-manifest.json" + platform_counts: Dict[str, int] = {} + for entry in entries: + platform_counts[entry["platform"]] = platform_counts.get(entry["platform"], 0) + 1 + missing_platforms = [ + platform + for platform in self._missing_symbol_platforms + if platform_counts.get(platform, 0) == 0 + ] + if missing_platforms: + raise RuntimeError( + "Debug extraction skipped all binaries for platforms without objcopy support: " + + ", ".join(sorted(missing_platforms)) + ) + manifest_data = OrderedDict(( + ("generatedAt", self.release_date), + ("version", self.version), + ("channel", self.channel), + ("artifacts", entries), + )) + with manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest_data, handle, indent=2) + handle.write("\n") + manifest_sha = compute_sha256(manifest_path) + sha_path = manifest_path.with_suffix(manifest_path.suffix + ".sha256") + sha_path.write_text(f"{manifest_sha} {manifest_path.name}\n", encoding="utf-8") + manifest_rel = manifest_path.relative_to(self.output_dir).as_posix() + store_rel = self.debug_store_dir.relative_to(self.output_dir).as_posix() + platforms = sorted({entry["platform"] for entry in entries}) + return OrderedDict(( + ("manifest", manifest_rel), + ("sha256", manifest_sha), + ("entries", len(entries)), + ("platforms", platforms), + ("directory", store_rel), + )) + + def _extract_debug_entries(self, component_name: str, image_ref: str) -> List[OrderedDict[str, Any]]: + if self.dry_run: + return [] + entries: List[OrderedDict[str, Any]] = [] + platforms = self.platforms if self.push else [None] + for platform in platforms: + platform_label = platform or (self.platforms[0] if self.platforms else "linux/amd64") + if self.push: + pull_cmd = ["docker", "pull"] + if platform: + pull_cmd.extend(["--platform", platform]) + pull_cmd.append(image_ref) + run(pull_cmd) + create_cmd = ["docker", "create"] + if platform: + create_cmd.extend(["--platform", platform]) + create_cmd.append(image_ref) + container_id = run(create_cmd).strip() + export_path = self.temp_dir / f"{container_id}.tar" + try: + run(["docker", "export", container_id, "-o", str(export_path)], capture=False) + finally: + run(["docker", "rm", container_id], capture=False) + rootfs_dir = ensure_directory(self.temp_dir / f"{component_name}-{platform_label}-{uuid.uuid4().hex}") + try: + with tarfile.open(export_path, "r:*") as tar: + self._safe_extract_tar(tar, rootfs_dir) + finally: + export_path.unlink(missing_ok=True) + try: + for file_path in rootfs_dir.rglob("*"): + if not file_path.is_file() or file_path.is_symlink(): + continue + if not self._is_elf(file_path): + continue + build_id, machine = self._read_build_id_and_machine(file_path) + if not build_id: + continue + debug_file = self._debug_file_for_build_id(build_id) + if not debug_file.exists(): + debug_file.parent.mkdir(parents=True, exist_ok=True) + temp_debug = self.temp_dir / f"{build_id}.debug" + with contextlib.suppress(FileNotFoundError): + temp_debug.unlink() + objcopy_tool = self._resolve_objcopy_tool(machine) + if not objcopy_tool: + self._emit_objcopy_warning(machine, platform_label, file_path) + with contextlib.suppress(FileNotFoundError): + temp_debug.unlink() + continue + try: + run([objcopy_tool, "--only-keep-debug", str(file_path), str(temp_debug)], capture=False) + except CommandError: + with contextlib.suppress(FileNotFoundError): + temp_debug.unlink() + continue + debug_file.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(temp_debug), str(debug_file)) + sha = compute_sha256(debug_file) + rel_debug = debug_file.relative_to(self.output_dir).as_posix() + source_rel = file_path.relative_to(rootfs_dir).as_posix() + entry = OrderedDict(( + ("component", component_name), + ("image", image_ref), + ("platform", platform_label), + ("buildId", build_id), + ("debugPath", rel_debug), + ("sha256", sha), + ("size", debug_file.stat().st_size), + ("sources", [source_rel]), + )) + entries.append(entry) + finally: + shutil.rmtree(rootfs_dir, ignore_errors=True) + return entries + + def _debug_file_for_build_id(self, build_id: str) -> pathlib.Path: + normalized = build_id.lower() + prefix = normalized[:2] + remainder = normalized[2:] + return self.debug_store_dir / prefix / f"{remainder}.debug" + + @staticmethod + def _safe_extract_tar(tar: tarfile.TarFile, dest: pathlib.Path) -> None: + dest_root = dest.resolve() + members = tar.getmembers() + for member in members: + member_path = (dest / member.name).resolve() + if not str(member_path).startswith(str(dest_root)): + raise RuntimeError(f"Refusing to extract '{member.name}' outside of destination directory") + tar.extractall(dest) + + @staticmethod + def _is_elf(path: pathlib.Path) -> bool: + try: + with path.open("rb") as handle: + return handle.read(4) == b"\x7fELF" + except OSError: + return False + + def _read_build_id_and_machine(self, path: pathlib.Path) -> Tuple[Optional[str], Optional[str]]: + try: + header_output = run(["readelf", "-nh", str(path)]) + except CommandError: + return None, None + build_id: Optional[str] = None + machine: Optional[str] = None + for line in header_output.splitlines(): + stripped = line.strip() + if stripped.startswith("Build ID:"): + build_id = stripped.split("Build ID:", 1)[1].strip().lower() + elif stripped.startswith("Machine:"): + machine = stripped.split("Machine:", 1)[1].strip() + return build_id, machine + + def _resolve_objcopy_tool(self, machine: Optional[str]) -> Optional[str]: + key = (machine or "generic").lower() + if key in self._objcopy_cache: + return self._objcopy_cache[key] + + env_override = None + if machine and "aarch64" in machine.lower(): + env_override = os.environ.get("STELLAOPS_OBJCOPY_AARCH64") + candidates = [ + env_override, + "aarch64-linux-gnu-objcopy", + "llvm-objcopy", + "objcopy", + ] + elif machine and any(token in machine.lower() for token in ("x86-64", "amd", "x86_64")): + env_override = os.environ.get("STELLAOPS_OBJCOPY_AMD64") + candidates = [ + env_override, + "objcopy", + "llvm-objcopy", + ] + else: + env_override = os.environ.get("STELLAOPS_OBJCOPY_DEFAULT") + candidates = [ + env_override, + "objcopy", + "llvm-objcopy", + ] + + for candidate in candidates: + if not candidate: + continue + tool = shutil.which(candidate) + if tool: + self._objcopy_cache[key] = tool + return tool + self._objcopy_cache[key] = None + return None + + def _emit_objcopy_warning(self, machine: Optional[str], platform: str, file_path: pathlib.Path) -> None: + machine_label = machine or "unknown-machine" + count = self._missing_symbol_platforms.get(platform, 0) + self._missing_symbol_platforms[platform] = count + 1 + if count == 0: + sys.stderr.write( + f"[warn] no objcopy tool available for {machine_label}; skipping debug extraction for {file_path}.\n" + ) + # ---------------- # Helm + compose # ---------------- @@ -546,6 +831,7 @@ class ReleaseBuilder: components: List[Mapping[str, Any]], helm_meta: Optional[Mapping[str, Any]], compose_meta: List[Mapping[str, Any]], + debug_meta: Optional[Mapping[str, Any]], ) -> Dict[str, Any]: manifest = OrderedDict() manifest["release"] = OrderedDict(( @@ -559,6 +845,8 @@ class ReleaseBuilder: manifest["charts"] = [helm_meta] if compose_meta: manifest["compose"] = compose_meta + if debug_meta: + manifest["debugStore"] = debug_meta return manifest @@ -593,6 +881,18 @@ def write_manifest(manifest: Mapping[str, Any], output_dir: pathlib.Path) -> pat output_path = output_dir / "release.yaml" with output_path.open("w", encoding="utf-8") as handle: handle.write(final_yaml) + sha_path = output_path.with_name(output_path.name + ".sha256") + yaml_file_digest = compute_sha256(output_path) + sha_path.write_text(f"{yaml_file_digest} {output_path.name}\n", encoding="utf-8") + + json_text = json.dumps(manifest_with_checksum, indent=2) + json_path = output_dir / "release.json" + with json_path.open("w", encoding="utf-8") as handle: + handle.write(json_text) + handle.write("\n") + json_digest = compute_sha256(json_path) + json_sha_path = json_path.with_name(json_path.name + ".sha256") + json_sha_path.write_text(f"{json_digest} {json_path.name}\n", encoding="utf-8") return output_path diff --git a/ops/devops/release/test_verify_release.py b/ops/devops/release/test_verify_release.py new file mode 100644 index 00000000..884a2c02 --- /dev/null +++ b/ops/devops/release/test_verify_release.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from collections import OrderedDict +from pathlib import Path +import sys + +sys.path.append(str(Path(__file__).resolve().parent)) + +from build_release import write_manifest # type: ignore import-not-found +from verify_release import VerificationError, compute_sha256, verify_release + + +class VerifyReleaseTests(unittest.TestCase): + def setUp(self) -> None: + self._temp = tempfile.TemporaryDirectory() + self.base_path = Path(self._temp.name) + self.out_dir = self.base_path / "out" + self.release_dir = self.out_dir / "release" + self.release_dir.mkdir(parents=True, exist_ok=True) + + def tearDown(self) -> None: + self._temp.cleanup() + + def _relative_to_out(self, path: Path) -> str: + return path.relative_to(self.out_dir).as_posix() + + def _write_json(self, path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) + handle.write("\n") + + def _create_sample_release(self) -> None: + sbom_path = self.release_dir / "artifacts/sboms/sample.cyclonedx.json" + sbom_path.parent.mkdir(parents=True, exist_ok=True) + sbom_path.write_text('{"bomFormat":"CycloneDX","specVersion":"1.5"}\n', encoding="utf-8") + sbom_sha = compute_sha256(sbom_path) + + provenance_path = self.release_dir / "artifacts/provenance/sample.provenance.json" + self._write_json( + provenance_path, + { + "buildDefinition": {"buildType": "https://example/build", "externalParameters": {}}, + "runDetails": {"builder": {"id": "https://example/ci"}}, + }, + ) + provenance_sha = compute_sha256(provenance_path) + + signature_path = self.release_dir / "artifacts/signatures/sample.signature" + signature_path.parent.mkdir(parents=True, exist_ok=True) + signature_path.write_text("signature-data\n", encoding="utf-8") + signature_sha = compute_sha256(signature_path) + + metadata_path = self.release_dir / "artifacts/metadata/sample.metadata.json" + self._write_json(metadata_path, {"digest": "sha256:1234"}) + metadata_sha = compute_sha256(metadata_path) + + chart_path = self.release_dir / "helm/stellaops-1.0.0.tgz" + chart_path.parent.mkdir(parents=True, exist_ok=True) + chart_path.write_bytes(b"helm-chart-data") + chart_sha = compute_sha256(chart_path) + + compose_path = self.release_dir.parent / "deploy/compose/docker-compose.dev.yaml" + compose_path.parent.mkdir(parents=True, exist_ok=True) + compose_path.write_text("services: {}\n", encoding="utf-8") + compose_sha = compute_sha256(compose_path) + + debug_file = self.release_dir / "debug/.build-id/ab/cdef.debug" + debug_file.parent.mkdir(parents=True, exist_ok=True) + debug_file.write_bytes(b"\x7fELFDEBUGDATA") + debug_sha = compute_sha256(debug_file) + + debug_manifest_path = self.release_dir / "debug/debug-manifest.json" + debug_manifest = OrderedDict( + ( + ("generatedAt", "2025-10-26T00:00:00Z"), + ("version", "1.0.0"), + ("channel", "edge"), + ( + "artifacts", + [ + OrderedDict( + ( + ("buildId", "abcdef1234"), + ("platform", "linux/amd64"), + ("debugPath", "debug/.build-id/ab/cdef.debug"), + ("sha256", debug_sha), + ("size", debug_file.stat().st_size), + ("components", ["sample"]), + ("images", ["registry.example/sample@sha256:feedface"]), + ("sources", ["app/sample.dll"]), + ) + ) + ], + ), + ) + ) + self._write_json(debug_manifest_path, debug_manifest) + debug_manifest_sha = compute_sha256(debug_manifest_path) + (debug_manifest_path.with_suffix(debug_manifest_path.suffix + ".sha256")).write_text( + f"{debug_manifest_sha} {debug_manifest_path.name}\n", encoding="utf-8" + ) + + manifest = OrderedDict( + ( + ( + "release", + OrderedDict( + ( + ("version", "1.0.0"), + ("channel", "edge"), + ("date", "2025-10-26T00:00:00Z"), + ("calendar", "2025.10"), + ) + ), + ), + ( + "components", + [ + OrderedDict( + ( + ("name", "sample"), + ("image", "registry.example/sample@sha256:feedface"), + ("tags", ["registry.example/sample:1.0.0"]), + ( + "sbom", + OrderedDict( + ( + ("path", self._relative_to_out(sbom_path)), + ("sha256", sbom_sha), + ) + ), + ), + ( + "provenance", + OrderedDict( + ( + ("path", self._relative_to_out(provenance_path)), + ("sha256", provenance_sha), + ) + ), + ), + ( + "signature", + OrderedDict( + ( + ("path", self._relative_to_out(signature_path)), + ("sha256", signature_sha), + ("ref", "sigstore://example"), + ("tlogUploaded", True), + ) + ), + ), + ( + "metadata", + OrderedDict( + ( + ("path", self._relative_to_out(metadata_path)), + ("sha256", metadata_sha), + ) + ), + ), + ) + ) + ], + ), + ( + "charts", + [ + OrderedDict( + ( + ("name", "stellaops"), + ("version", "1.0.0"), + ("path", self._relative_to_out(chart_path)), + ("sha256", chart_sha), + ) + ) + ], + ), + ( + "compose", + [ + OrderedDict( + ( + ("name", "docker-compose.dev.yaml"), + ("path", compose_path.relative_to(self.out_dir).as_posix()), + ("sha256", compose_sha), + ) + ) + ], + ), + ( + "debugStore", + OrderedDict( + ( + ("manifest", "debug/debug-manifest.json"), + ("sha256", debug_manifest_sha), + ("entries", 1), + ("platforms", ["linux/amd64"]), + ("directory", "debug/.build-id"), + ) + ), + ), + ) + ) + write_manifest(manifest, self.release_dir) + + def test_verify_release_success(self) -> None: + self._create_sample_release() + # Should not raise + verify_release(self.release_dir) + + def test_verify_release_detects_sha_mismatch(self) -> None: + self._create_sample_release() + tampered = self.release_dir / "artifacts/sboms/sample.cyclonedx.json" + tampered.write_text("tampered\n", encoding="utf-8") + with self.assertRaises(VerificationError): + verify_release(self.release_dir) + + def test_verify_release_detects_missing_debug_file(self) -> None: + self._create_sample_release() + debug_file = self.release_dir / "debug/.build-id/ab/cdef.debug" + debug_file.unlink() + with self.assertRaises(VerificationError): + verify_release(self.release_dir) + + +if __name__ == "__main__": + unittest.main() diff --git a/ops/devops/release/verify_release.py b/ops/devops/release/verify_release.py new file mode 100644 index 00000000..f396d86e --- /dev/null +++ b/ops/devops/release/verify_release.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +"""Verify release artefacts (SBOMs, provenance, signatures, manifest hashes).""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import pathlib +import sys +from collections import OrderedDict +from typing import Any, Mapping, Optional + +from build_release import dump_yaml # type: ignore import-not-found + + +class VerificationError(Exception): + """Raised when release artefacts fail verification.""" + + +def compute_sha256(path: pathlib.Path) -> str: + sha = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(chunk) + return sha.hexdigest() + + +def parse_sha_file(path: pathlib.Path) -> Optional[str]: + if not path.exists(): + return None + content = path.read_text(encoding="utf-8").strip() + if not content: + return None + return content.split()[0] + + +def resolve_path(path_str: str, release_dir: pathlib.Path) -> pathlib.Path: + candidate = pathlib.Path(path_str.replace("\\", "/")) + if candidate.is_absolute(): + return candidate + + for base in (release_dir, release_dir.parent, release_dir.parent.parent): + resolved = (base / candidate).resolve() + if resolved.exists(): + return resolved + # Fall back to release_dir joined path even if missing to surface in caller. + return (release_dir / candidate).resolve() + + +def load_manifest(release_dir: pathlib.Path) -> OrderedDict[str, Any]: + manifest_path = release_dir / "release.json" + if not manifest_path.exists(): + raise VerificationError(f"Release manifest JSON missing at {manifest_path}") + try: + with manifest_path.open("r", encoding="utf-8") as handle: + return json.load(handle, object_pairs_hook=OrderedDict) + except json.JSONDecodeError as exc: + raise VerificationError(f"Failed to parse {manifest_path}: {exc}") from exc + + +def verify_manifest_hashes( + manifest: Mapping[str, Any], + release_dir: pathlib.Path, + errors: list[str], +) -> None: + yaml_path = release_dir / "release.yaml" + if not yaml_path.exists(): + errors.append(f"Missing release.yaml at {yaml_path}") + return + + recorded_yaml_sha = parse_sha_file(yaml_path.with_name(yaml_path.name + ".sha256")) + actual_yaml_sha = compute_sha256(yaml_path) + if recorded_yaml_sha and recorded_yaml_sha != actual_yaml_sha: + errors.append( + f"release.yaml.sha256 recorded {recorded_yaml_sha} but file hashes to {actual_yaml_sha}" + ) + + json_path = release_dir / "release.json" + recorded_json_sha = parse_sha_file(json_path.with_name(json_path.name + ".sha256")) + actual_json_sha = compute_sha256(json_path) + if recorded_json_sha and recorded_json_sha != actual_json_sha: + errors.append( + f"release.json.sha256 recorded {recorded_json_sha} but file hashes to {actual_json_sha}" + ) + + checksums = manifest.get("checksums") + if isinstance(checksums, Mapping): + recorded_digest = checksums.get("sha256") + base_manifest = OrderedDict(manifest) + base_manifest.pop("checksums", None) + yaml_without_checksums = dump_yaml(base_manifest) + computed_digest = hashlib.sha256(yaml_without_checksums.encode("utf-8")).hexdigest() + if recorded_digest != computed_digest: + errors.append( + "Manifest checksum mismatch: " + f"recorded {recorded_digest}, computed {computed_digest}" + ) + + +def verify_artifact_entry( + entry: Mapping[str, Any], + release_dir: pathlib.Path, + label: str, + component_name: str, + errors: list[str], +) -> None: + path_str = entry.get("path") + if not path_str: + errors.append(f"{component_name}: {label} missing 'path' field.") + return + resolved = resolve_path(str(path_str), release_dir) + if not resolved.exists(): + errors.append(f"{component_name}: {label} path does not exist → {resolved}") + return + recorded_sha = entry.get("sha256") + if recorded_sha: + actual_sha = compute_sha256(resolved) + if actual_sha != recorded_sha: + errors.append( + f"{component_name}: {label} SHA mismatch for {resolved} " + f"(recorded {recorded_sha}, computed {actual_sha})" + ) + + +def verify_components(manifest: Mapping[str, Any], release_dir: pathlib.Path, errors: list[str]) -> None: + for component in manifest.get("components", []): + if not isinstance(component, Mapping): + errors.append("Component entry is not a mapping.") + continue + name = str(component.get("name", "")) + for key, label in ( + ("sbom", "SBOM"), + ("provenance", "provenance"), + ("signature", "signature"), + ("metadata", "metadata"), + ): + entry = component.get(key) + if not entry: + continue + if not isinstance(entry, Mapping): + errors.append(f"{name}: {label} entry must be a mapping.") + continue + verify_artifact_entry(entry, release_dir, label, name, errors) + + +def verify_collections(manifest: Mapping[str, Any], release_dir: pathlib.Path, errors: list[str]) -> None: + for collection, label in ( + ("charts", "chart"), + ("compose", "compose file"), + ): + for item in manifest.get(collection, []): + if not isinstance(item, Mapping): + errors.append(f"{collection} entry is not a mapping.") + continue + path_value = item.get("path") + if not path_value: + errors.append(f"{collection} entry missing path.") + continue + resolved = resolve_path(str(path_value), release_dir) + if not resolved.exists(): + errors.append(f"{label} missing file → {resolved}") + continue + recorded_sha = item.get("sha256") + if recorded_sha: + actual_sha = compute_sha256(resolved) + if actual_sha != recorded_sha: + errors.append( + f"{label} SHA mismatch for {resolved} " + f"(recorded {recorded_sha}, computed {actual_sha})" + ) + + +def verify_debug_store(manifest: Mapping[str, Any], release_dir: pathlib.Path, errors: list[str]) -> None: + debug = manifest.get("debugStore") + if not isinstance(debug, Mapping): + return + manifest_path_str = debug.get("manifest") + manifest_data: Optional[Mapping[str, Any]] = None + if manifest_path_str: + manifest_path = resolve_path(str(manifest_path_str), release_dir) + if not manifest_path.exists(): + errors.append(f"Debug manifest missing → {manifest_path}") + else: + recorded_sha = debug.get("sha256") + if recorded_sha: + actual_sha = compute_sha256(manifest_path) + if actual_sha != recorded_sha: + errors.append( + f"Debug manifest SHA mismatch (recorded {recorded_sha}, computed {actual_sha})" + ) + sha_sidecar = manifest_path.with_suffix(manifest_path.suffix + ".sha256") + sidecar_sha = parse_sha_file(sha_sidecar) + if sidecar_sha and recorded_sha and sidecar_sha != recorded_sha: + errors.append( + f"Debug manifest sidecar digest {sidecar_sha} disagrees with recorded {recorded_sha}" + ) + try: + with manifest_path.open("r", encoding="utf-8") as handle: + manifest_data = json.load(handle) + except json.JSONDecodeError as exc: + errors.append(f"Debug manifest JSON invalid: {exc}") + directory = debug.get("directory") + if directory: + debug_dir = resolve_path(str(directory), release_dir) + if not debug_dir.exists(): + errors.append(f"Debug directory missing → {debug_dir}") + + if manifest_data: + artifacts = manifest_data.get("artifacts") + if not isinstance(artifacts, list) or not artifacts: + errors.append("Debug manifest contains no artefacts.") + return + + declared_entries = debug.get("entries") + if isinstance(declared_entries, int) and declared_entries != len(artifacts): + errors.append( + f"Debug manifest reports {declared_entries} entries but contains {len(artifacts)} artefacts." + ) + + for artefact in artifacts: + if not isinstance(artefact, Mapping): + errors.append("Debug manifest artefact entry is not a mapping.") + continue + debug_path = artefact.get("debugPath") + artefact_sha = artefact.get("sha256") + if not debug_path or not artefact_sha: + errors.append("Debug manifest artefact missing debugPath or sha256.") + continue + resolved_debug = resolve_path(str(debug_path), release_dir) + if not resolved_debug.exists(): + errors.append(f"Debug artefact missing → {resolved_debug}") + continue + actual_sha = compute_sha256(resolved_debug) + if actual_sha != artefact_sha: + errors.append( + f"Debug artefact SHA mismatch for {resolved_debug} " + f"(recorded {artefact_sha}, computed {actual_sha})" + ) + + +def verify_release(release_dir: pathlib.Path) -> None: + if not release_dir.exists(): + raise VerificationError(f"Release directory not found: {release_dir}") + manifest = load_manifest(release_dir) + errors: list[str] = [] + verify_manifest_hashes(manifest, release_dir, errors) + verify_components(manifest, release_dir, errors) + verify_collections(manifest, release_dir, errors) + verify_debug_store(manifest, release_dir, errors) + if errors: + bullet_list = "\n - ".join(errors) + raise VerificationError(f"Release verification failed:\n - {bullet_list}") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--release-dir", + type=pathlib.Path, + default=pathlib.Path("out/release"), + help="Path to the release artefact directory (default: %(default)s)", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + verify_release(args.release_dir.resolve()) + except VerificationError as exc: + print(str(exc), file=sys.stderr) + return 1 + print(f"✅ Release artefacts verified OK in {args.release_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devops/telemetry/__pycache__/package_offline_bundle.cpython-312.pyc b/ops/devops/telemetry/__pycache__/package_offline_bundle.cpython-312.pyc new file mode 100644 index 00000000..68c232d6 Binary files /dev/null and b/ops/devops/telemetry/__pycache__/package_offline_bundle.cpython-312.pyc differ diff --git a/ops/devops/telemetry/generate_dev_tls.sh b/ops/devops/telemetry/generate_dev_tls.sh new file mode 100644 index 00000000..348a3516 --- /dev/null +++ b/ops/devops/telemetry/generate_dev_tls.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CERT_DIR="${SCRIPT_DIR}/../../deploy/telemetry/certs" + +mkdir -p "${CERT_DIR}" + +CA_KEY="${CERT_DIR}/ca.key" +CA_CRT="${CERT_DIR}/ca.crt" +COL_KEY="${CERT_DIR}/collector.key" +COL_CSR="${CERT_DIR}/collector.csr" +COL_CRT="${CERT_DIR}/collector.crt" +CLIENT_KEY="${CERT_DIR}/client.key" +CLIENT_CSR="${CERT_DIR}/client.csr" +CLIENT_CRT="${CERT_DIR}/client.crt" + +echo "[*] Generating OpenTelemetry dev CA and certificates in ${CERT_DIR}" + +# Root CA +if [[ ! -f "${CA_KEY}" ]]; then + openssl genrsa -out "${CA_KEY}" 4096 >/dev/null 2>&1 +fi +openssl req -x509 -new -key "${CA_KEY}" -days 365 -sha256 \ + -out "${CA_CRT}" -subj "/CN=StellaOps Dev Telemetry CA" \ + -config <(cat <<'EOF' +[req] +distinguished_name = req_distinguished_name +prompt = no +[req_distinguished_name] +EOF +) >/dev/null 2>&1 + +# Collector certificate (server + client auth) +openssl req -new -nodes -newkey rsa:4096 \ + -keyout "${COL_KEY}" \ + -out "${COL_CSR}" \ + -subj "/CN=stellaops-otel-collector" >/dev/null 2>&1 + +openssl x509 -req -in "${COL_CSR}" -CA "${CA_CRT}" -CAkey "${CA_KEY}" \ + -CAcreateserial -out "${COL_CRT}" -days 365 -sha256 \ + -extensions v3_req -extfile <(cat <<'EOF' +[v3_req] +subjectAltName = @alt_names +extendedKeyUsage = serverAuth, clientAuth +[alt_names] +DNS.1 = stellaops-otel-collector +DNS.2 = localhost +IP.1 = 127.0.0.1 +EOF +) >/dev/null 2>&1 + +# Client certificate +openssl req -new -nodes -newkey rsa:4096 \ + -keyout "${CLIENT_KEY}" \ + -out "${CLIENT_CSR}" \ + -subj "/CN=stellaops-otel-client" >/dev/null 2>&1 + +openssl x509 -req -in "${CLIENT_CSR}" -CA "${CA_CRT}" -CAkey "${CA_KEY}" \ + -CAcreateserial -out "${CLIENT_CRT}" -days 365 -sha256 \ + -extensions v3_req -extfile <(cat <<'EOF' +[v3_req] +extendedKeyUsage = clientAuth +subjectAltName = @alt_names +[alt_names] +DNS.1 = stellaops-otel-client +DNS.2 = localhost +IP.1 = 127.0.0.1 +EOF +) >/dev/null 2>&1 + +rm -f "${COL_CSR}" "${CLIENT_CSR}" +rm -f "${CERT_DIR}/ca.srl" + +echo "[✓] Certificates ready:" +ls -1 "${CERT_DIR}" diff --git a/ops/devops/telemetry/package_offline_bundle.py b/ops/devops/telemetry/package_offline_bundle.py new file mode 100644 index 00000000..28ea0908 --- /dev/null +++ b/ops/devops/telemetry/package_offline_bundle.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Package telemetry collector assets for offline/air-gapped installs. + +Outputs a tarball containing the collector configuration, Compose overlay, +Helm defaults, and operator README. A SHA-256 checksum sidecar is emitted, and +optional Cosign signing can be enabled with --sign. +""" +from __future__ import annotations + +import argparse +import hashlib +import os +import subprocess +import sys +import tarfile +from pathlib import Path +from typing import Iterable + +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_OUTPUT = REPO_ROOT / "out" / "telemetry" / "telemetry-offline-bundle.tar.gz" +BUNDLE_CONTENTS: tuple[Path, ...] = ( + Path("deploy/telemetry/README.md"), + Path("deploy/telemetry/otel-collector-config.yaml"), + Path("deploy/telemetry/storage/README.md"), + Path("deploy/telemetry/storage/prometheus.yaml"), + Path("deploy/telemetry/storage/tempo.yaml"), + Path("deploy/telemetry/storage/loki.yaml"), + Path("deploy/telemetry/storage/tenants/tempo-overrides.yaml"), + Path("deploy/telemetry/storage/tenants/loki-overrides.yaml"), + Path("deploy/helm/stellaops/files/otel-collector-config.yaml"), + Path("deploy/helm/stellaops/values.yaml"), + Path("deploy/helm/stellaops/templates/otel-collector.yaml"), + Path("deploy/compose/docker-compose.telemetry.yaml"), + Path("deploy/compose/docker-compose.telemetry-storage.yaml"), + Path("docs/ops/telemetry-collector.md"), + Path("docs/ops/telemetry-storage.md"), +) + + +def compute_sha256(path: Path) -> str: + sha = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(chunk) + return sha.hexdigest() + + +def validate_files(paths: Iterable[Path]) -> None: + missing = [str(p) for p in paths if not (REPO_ROOT / p).exists()] + if missing: + raise FileNotFoundError(f"Missing bundle artefacts: {', '.join(missing)}") + + +def create_bundle(output_path: Path) -> Path: + output_path.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(output_path, "w:gz") as tar: + for rel_path in BUNDLE_CONTENTS: + abs_path = REPO_ROOT / rel_path + tar.add(abs_path, arcname=str(rel_path)) + return output_path + + +def write_checksum(bundle_path: Path) -> Path: + digest = compute_sha256(bundle_path) + sha_path = bundle_path.with_suffix(bundle_path.suffix + ".sha256") + sha_path.write_text(f"{digest} {bundle_path.name}\n", encoding="utf-8") + return sha_path + + +def cosign_sign(bundle_path: Path, key_ref: str | None, identity_token: str | None) -> None: + cmd = ["cosign", "sign-blob", "--yes", str(bundle_path)] + if key_ref: + cmd.extend(["--key", key_ref]) + env = os.environ.copy() + if identity_token: + env["COSIGN_IDENTITY_TOKEN"] = identity_token + try: + subprocess.run(cmd, check=True, env=env) + except FileNotFoundError as exc: + raise RuntimeError("cosign not found on PATH; install cosign or omit --sign") from exc + except subprocess.CalledProcessError as exc: + raise RuntimeError(f"cosign sign-blob failed: {exc}") from exc + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + type=Path, + default=DEFAULT_OUTPUT, + help=f"Output bundle path (default: {DEFAULT_OUTPUT})", + ) + parser.add_argument( + "--sign", + action="store_true", + help="Sign the bundle using cosign (requires cosign on PATH)", + ) + parser.add_argument( + "--cosign-key", + type=str, + default=os.environ.get("COSIGN_KEY_REF"), + help="Cosign key reference (file:..., azurekms://..., etc.)", + ) + parser.add_argument( + "--identity-token", + type=str, + default=os.environ.get("COSIGN_IDENTITY_TOKEN"), + help="OIDC identity token for keyless signing", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + validate_files(BUNDLE_CONTENTS) + + bundle_path = args.output.resolve() + print(f"[*] Creating telemetry bundle at {bundle_path}") + create_bundle(bundle_path) + sha_path = write_checksum(bundle_path) + print(f"[✓] SHA-256 written to {sha_path}") + + if args.sign: + print("[*] Signing bundle with cosign") + cosign_sign(bundle_path, args.cosign_key, args.identity_token) + sig_path = bundle_path.with_suffix(bundle_path.suffix + ".sig") + if sig_path.exists(): + print(f"[✓] Cosign signature written to {sig_path}") + else: + print("[!] Cosign completed but signature file not found (ensure cosign version >= 2.2)") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ops/devops/telemetry/smoke_otel_collector.py b/ops/devops/telemetry/smoke_otel_collector.py new file mode 100644 index 00000000..771fa986 --- /dev/null +++ b/ops/devops/telemetry/smoke_otel_collector.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +""" +Smoke test for the StellaOps OpenTelemetry Collector deployment. + +The script sends sample traces, metrics, and logs over OTLP/HTTP with mutual TLS +and asserts that the collector accepted the payloads by checking its Prometheus +metrics endpoint. +""" + +from __future__ import annotations + +import argparse +import json +import ssl +import sys +import time +import urllib.request +from pathlib import Path + +TRACE_PAYLOAD = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "smoke-client"}}, + {"key": "tenant.id", "value": {"stringValue": "dev"}}, + ] + }, + "scopeSpans": [ + { + "scope": {"name": "smoke-test"}, + "spans": [ + { + "traceId": "00000000000000000000000000000001", + "spanId": "0000000000000001", + "name": "smoke-span", + "kind": 1, + "startTimeUnixNano": "1730000000000000000", + "endTimeUnixNano": "1730000000500000000", + "status": {"code": 0}, + } + ], + } + ], + } + ] +} + +METRIC_PAYLOAD = { + "resourceMetrics": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "smoke-client"}}, + {"key": "tenant.id", "value": {"stringValue": "dev"}}, + ] + }, + "scopeMetrics": [ + { + "scope": {"name": "smoke-test"}, + "metrics": [ + { + "name": "smoke_gauge", + "gauge": { + "dataPoints": [ + { + "asDouble": 1.0, + "timeUnixNano": "1730000001000000000", + "attributes": [ + {"key": "phase", "value": {"stringValue": "ingest"}} + ], + } + ] + }, + } + ], + } + ], + } + ] +} + +LOG_PAYLOAD = { + "resourceLogs": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "smoke-client"}}, + {"key": "tenant.id", "value": {"stringValue": "dev"}}, + ] + }, + "scopeLogs": [ + { + "scope": {"name": "smoke-test"}, + "logRecords": [ + { + "timeUnixNano": "1730000002000000000", + "severityNumber": 9, + "severityText": "Info", + "body": {"stringValue": "StellaOps collector smoke log"}, + } + ], + } + ], + } + ] +} + + +def _load_context(ca: Path, cert: Path, key: Path) -> ssl.SSLContext: + context = ssl.create_default_context(cafile=str(ca)) + context.check_hostname = False + context.verify_mode = ssl.CERT_REQUIRED + context.load_cert_chain(certfile=str(cert), keyfile=str(key)) + return context + + +def _post_json(url: str, payload: dict, context: ssl.SSLContext) -> None: + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + url, + data=data, + headers={ + "Content-Type": "application/json", + "User-Agent": "stellaops-otel-smoke/1.0", + }, + method="POST", + ) + with urllib.request.urlopen(request, context=context, timeout=10) as response: + if response.status // 100 != 2: + raise RuntimeError(f"{url} returned HTTP {response.status}") + + +def _fetch_metrics(url: str, context: ssl.SSLContext) -> str: + request = urllib.request.Request( + url, + headers={ + "User-Agent": "stellaops-otel-smoke/1.0", + }, + ) + with urllib.request.urlopen(request, context=context, timeout=10) as response: + return response.read().decode("utf-8") + + +def _assert_counter(metrics: str, metric_name: str) -> None: + for line in metrics.splitlines(): + if line.startswith(metric_name): + try: + _, value = line.split(" ") + if float(value) > 0: + return + except ValueError: + continue + raise AssertionError(f"{metric_name} not incremented") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", default="localhost", help="Collector host (default: %(default)s)") + parser.add_argument("--otlp-port", type=int, default=4318, help="OTLP/HTTP port") + parser.add_argument("--metrics-port", type=int, default=9464, help="Prometheus metrics port") + parser.add_argument("--health-port", type=int, default=13133, help="Health check port") + parser.add_argument("--ca", type=Path, default=Path("deploy/telemetry/certs/ca.crt"), help="CA certificate path") + parser.add_argument("--cert", type=Path, default=Path("deploy/telemetry/certs/client.crt"), help="Client certificate path") + parser.add_argument("--key", type=Path, default=Path("deploy/telemetry/certs/client.key"), help="Client key path") + args = parser.parse_args() + + for path in (args.ca, args.cert, args.key): + if not path.exists(): + print(f"[!] missing TLS material: {path}", file=sys.stderr) + return 1 + + context = _load_context(args.ca, args.cert, args.key) + + otlp_base = f"https://{args.host}:{args.otlp_port}/v1" + print(f"[*] Sending OTLP traffic to {otlp_base}") + _post_json(f"{otlp_base}/traces", TRACE_PAYLOAD, context) + _post_json(f"{otlp_base}/metrics", METRIC_PAYLOAD, context) + _post_json(f"{otlp_base}/logs", LOG_PAYLOAD, context) + + # Allow Prometheus exporter to update metrics + time.sleep(2) + + metrics_url = f"https://{args.host}:{args.metrics_port}/metrics" + print(f"[*] Fetching collector metrics from {metrics_url}") + metrics = _fetch_metrics(metrics_url, context) + + _assert_counter(metrics, "otelcol_receiver_accepted_spans") + _assert_counter(metrics, "otelcol_receiver_accepted_logs") + _assert_counter(metrics, "otelcol_receiver_accepted_metric_points") + + print("[✓] Collector accepted traces, logs, and metrics.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/devops/validate_restore_sources.py b/ops/devops/validate_restore_sources.py new file mode 100644 index 00000000..06bb2bc5 --- /dev/null +++ b/ops/devops/validate_restore_sources.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 + +""" +Validate NuGet source ordering for StellaOps. + +Ensures `local-nuget` is the highest priority feed in both NuGet.config and the +Directory.Build.props restore configuration. Fails fast with actionable errors +so CI/offline kit workflows can assert deterministic restore ordering. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +NUGET_CONFIG = REPO_ROOT / "NuGet.config" +ROOT_PROPS = REPO_ROOT / "Directory.Build.props" +EXPECTED_SOURCE_KEYS = ["local", "dotnet-public", "nuget.org"] + + +class ValidationError(Exception): + """Raised when validation fails.""" + + +def _fail(message: str) -> None: + raise ValidationError(message) + + +def _parse_xml(path: Path) -> ET.ElementTree: + try: + return ET.parse(path) + except FileNotFoundError as exc: + _fail(f"Missing required file: {path}") + except ET.ParseError as exc: + _fail(f"Could not parse XML for {path}: {exc}") + + +def validate_nuget_config() -> None: + tree = _parse_xml(NUGET_CONFIG) + root = tree.getroot() + + package_sources = root.find("packageSources") + if package_sources is None: + _fail("NuGet.config must declare a section.") + + children = list(package_sources) + if not children or children[0].tag != "clear": + _fail("NuGet.config packageSources must begin with a element.") + + adds = [child for child in children if child.tag == "add"] + if not adds: + _fail("NuGet.config packageSources must define at least one entry.") + + keys = [add.attrib.get("key") for add in adds] + if keys[: len(EXPECTED_SOURCE_KEYS)] != EXPECTED_SOURCE_KEYS: + formatted = ", ".join(keys) or "" + _fail( + "NuGet.config packageSources must list feeds in the order " + f"{EXPECTED_SOURCE_KEYS}. Found: {formatted}" + ) + + local_value = adds[0].attrib.get("value", "") + if Path(local_value).name != "local-nuget": + _fail( + "NuGet.config local feed should point at the repo-local mirror " + f"'local-nuget', found value '{local_value}'." + ) + + clear = package_sources.find("clear") + if clear is None: + _fail("NuGet.config packageSources must start with to avoid inherited feeds.") + + +def validate_directory_build_props() -> None: + tree = _parse_xml(ROOT_PROPS) + root = tree.getroot() + defaults = None + for element in root.findall(".//_StellaOpsDefaultRestoreSources"): + defaults = [fragment.strip() for fragment in element.text.split(";") if fragment.strip()] + break + + if defaults is None: + _fail("Directory.Build.props must define _StellaOpsDefaultRestoreSources.") + + expected_props = [ + "$(StellaOpsLocalNuGetSource)", + "$(StellaOpsDotNetPublicSource)", + "$(StellaOpsNuGetOrgSource)", + ] + if defaults != expected_props: + _fail( + "Directory.Build.props _StellaOpsDefaultRestoreSources must list feeds " + f"in the order {expected_props}. Found: {defaults}" + ) + + restore_nodes = root.findall(".//RestoreSources") + if not restore_nodes: + _fail("Directory.Build.props must override RestoreSources to force deterministic ordering.") + + uses_default_first = any( + node.text + and node.text.strip().startswith("$(_StellaOpsDefaultRestoreSources)") + for node in restore_nodes + ) + if not uses_default_first: + _fail( + "Directory.Build.props RestoreSources override must place " + "$(_StellaOpsDefaultRestoreSources) at the beginning." + ) + + +def assert_single_nuget_config() -> None: + extra_configs: list[Path] = [] + configs: set[Path] = set() + for glob in ("NuGet.config", "nuget.config"): + try: + result = subprocess.run( + ["rg", "--files", f"-g{glob}"], + check=False, + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + except FileNotFoundError as exc: + _fail("ripgrep (rg) is required for validation but was not found on PATH.") + if result.returncode not in (0, 1): + _fail( + f"ripgrep failed while searching for {glob}: {result.stderr.strip() or result.returncode}" + ) + for line in result.stdout.splitlines(): + configs.add((REPO_ROOT / line).resolve()) + + configs.discard(NUGET_CONFIG.resolve()) + extra_configs.extend(sorted(configs)) + if extra_configs: + formatted = "\n ".join(str(path.relative_to(REPO_ROOT)) for path in extra_configs) + _fail( + "Unexpected additional NuGet.config files detected. " + "Consolidate feed configuration in the repo root:\n " + f"{formatted}" + ) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Verify StellaOps NuGet feeds prioritise the local mirror." + ) + parser.add_argument( + "--skip-rg", + action="store_true", + help="Skip ripgrep discovery of extra NuGet.config files (useful for focused runs).", + ) + return parser.parse_args(argv) + + +def main(argv: list[str]) -> int: + args = parse_args(argv) + validations = [ + ("NuGet.config ordering", validate_nuget_config), + ("Directory.Build.props restore override", validate_directory_build_props), + ] + if not args.skip_rg: + validations.append(("single NuGet.config", assert_single_nuget_config)) + + for label, check in validations: + try: + check() + except ValidationError as exc: + sys.stderr.write(f"[FAIL] {label}: {exc}\n") + return 1 + else: + sys.stdout.write(f"[OK] {label}\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/ops/licensing/TASKS.md b/ops/licensing/TASKS.md index 5c9a727c..08d7b4fb 100644 --- a/ops/licensing/TASKS.md +++ b/ops/licensing/TASKS.md @@ -2,4 +2,4 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DEVOPS-LIC-14-004 | TODO | Licensing Guild | AUTH-MTLS-11-002 | Implement registry token service tied to Authority (DPoP/mTLS), plan gating, revocation handling, and monitoring per architecture. | Token service issues scoped tokens, revocation tested, monitoring dashboards in place, docs updated. | +| DEVOPS-LIC-14-004 | DONE (2025-10-26) | Licensing Guild | AUTH-MTLS-11-002 | Implement registry token service tied to Authority (DPoP/mTLS), plan gating, revocation handling, and monitoring per architecture. | Token service issues scoped tokens, revocation tested, monitoring dashboards in place, docs updated. | diff --git a/ops/offline-kit/TASKS.md b/ops/offline-kit/TASKS.md index 1d144994..f3c85a8e 100644 --- a/ops/offline-kit/TASKS.md +++ b/ops/offline-kit/TASKS.md @@ -2,10 +2,13 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| DEVOPS-OFFLINE-14-002 | TODO | Offline Kit Guild | DEVOPS-REL-14-001 | Build offline kit packaging workflow (artifact bundling, manifest generation, signature verification). | Offline tarball generated with manifest + checksums + signatures; import script verifies integrity; docs updated. | +| DEVOPS-OFFLINE-14-002 | DONE (2025-10-26) | Offline Kit Guild | DEVOPS-REL-14-001 | Build offline kit packaging workflow (artifact bundling, manifest generation, signature verification). | Offline tarball generated with manifest + checksums + signatures; `ops/offline-kit/run-python-analyzer-smoke.sh` invoked as part of packaging; `debug/.build-id` tree mirrored from release output; import script verifies integrity; docs updated. | | DEVOPS-OFFLINE-18-004 | DONE (2025-10-22) | Offline Kit Guild, Scanner Guild | DEVOPS-OFFLINE-18-003, SCANNER-ANALYZERS-LANG-10-309G | Rebuild Offline Kit bundle with Go analyzer plug-in and updated manifest/signature set. | Kit tarball includes Go analyzer artifacts; manifest/signature refreshed; verification steps executed and logged; docs updated with new bundle version. | -| DEVOPS-OFFLINE-18-005 | TODO | Offline Kit Guild, Scanner Guild | DEVOPS-REL-14-004, SCANNER-ANALYZERS-LANG-10-309P | Repackage Offline Kit with Python analyzer plug-in artefacts and refreshed manifest/signature set. | Kit tarball includes Python analyzer DLL/PDB/manifest; signature + manifest updated; Offline Kit guide references Python coverage; smoke import validated. | +| DEVOPS-OFFLINE-18-005 | DONE (2025-10-26) | Offline Kit Guild, Scanner Guild | DEVOPS-REL-14-004, SCANNER-ANALYZERS-LANG-10-309P | Repackage Offline Kit with Python analyzer plug-in artefacts and refreshed manifest/signature set. | Kit tarball includes Python analyzer DLL/PDB/manifest; signature + manifest updated; Offline Kit guide references Python coverage; smoke import validated. | | DEVOPS-OFFLINE-34-006 | TODO | Offline Kit Guild, Orchestrator Service Guild | ORCH-SVC-34-004, DEPLOY-ORCH-34-001 | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. | Offline kit contains orchestrator assets; manifest/signature validated; docs updated with air-gapped install steps; smoke import executed. | -| DEVOPS-OFFLINE-37-001 | TODO | Offline Kit Guild, Exporter Service Guild | EXPORT-SVC-37-001..004, DEPLOY-EXPORT-36-001 | Package Export Center tooling, sample mirror bundles, verification CLI, and docs into Offline Kit with manifest/signature refresh and air-gap import script. | Offline kit includes export bundles/tools; verification script passes; manifest/signature updated; docs detail import workflow. | +| DEVOPS-OFFLINE-37-001 | TODO | Offline Kit Guild, Exporter Service Guild | EXPORT-SVC-37-001..004, DEPLOY-EXPORT-36-001 | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). | Offline kit includes export bundles/tools; verification script passes; manifest/signature updated; docs detail import workflow. | +| DEVOPS-OFFLINE-37-002 | TODO | Offline Kit Guild, Notifications Service Guild | NOTIFY-SVC-40-001..004, WEB-NOTIFY-40-001 | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. | Offline kit ships notifier assets with checksums; dry-run harness validated; docs outline sealed/connected install steps. | | CLI-PACKS-43-002 | TODO | Offline Kit Guild, Packs Registry Guild | PACKS-REG-42-001, DEPLOY-PACKS-43-001 | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline kit includes packs registry mirror, Task Runner configs, CLI binaries; manifest/signature updated; docs describe air-gapped execution. | | OFFLINE-CONTAINERS-46-001 | TODO | Offline Kit Guild, Deployment Guild | DEVOPS-CONTAINERS-46-001, DEPLOY-AIRGAP-46-001 | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline kit ships bundle + how-to; verification steps validated; manifest/signature updated; imposed rule noted. | +| DEVOPS-OFFLINE-17-003 | DONE (2025-10-26) | Offline Kit Guild, DevOps Guild | DEVOPS-REL-17-002 | Mirror release debug-store artefacts ( `.build-id/` tree and `debug-manifest.json`) into Offline Kit packaging and document import validation. | Offline kit archives `debug/.build-id/` with manifest/sha256, docs cover symbol lookup workflow, smoke job confirms build-id lookup succeeds on air-gapped install. | +| DEVOPS-OFFLINE-17-004 | BLOCKED (2025-10-26) | Offline Kit Guild, DevOps Guild | DEVOPS-REL-17-002 | Execute `mirror_debug_store.py` after the next release pipeline emits `out/release/debug`, verify manifest hashes, and archive `metadata/debug-store.json` with the kit. | Debug store mirrored post-release, manifest SHA validated, summary committed alongside Offline Kit bundle evidence. ⏳ Blocked until the release pipeline publishes the next `out/release/debug` tree; rerun the mirroring script as part of that pipeline. | diff --git a/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc b/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc new file mode 100644 index 00000000..356d4793 Binary files /dev/null and b/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc differ diff --git a/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc b/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc new file mode 100644 index 00000000..3a21c287 Binary files /dev/null and b/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc differ diff --git a/ops/offline-kit/__pycache__/test_build_offline_kit.cpython-312.pyc b/ops/offline-kit/__pycache__/test_build_offline_kit.cpython-312.pyc new file mode 100644 index 00000000..517ca94d Binary files /dev/null and b/ops/offline-kit/__pycache__/test_build_offline_kit.cpython-312.pyc differ diff --git a/ops/offline-kit/build_offline_kit.py b/ops/offline-kit/build_offline_kit.py new file mode 100644 index 00000000..8b36945e --- /dev/null +++ b/ops/offline-kit/build_offline_kit.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +"""Package the StellaOps Offline Kit with deterministic artefacts and manifest.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +from collections import OrderedDict +from pathlib import Path +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +REPO_ROOT = Path(__file__).resolve().parents[2] +RELEASE_TOOLS_DIR = REPO_ROOT / "ops" / "devops" / "release" +TELEMETRY_TOOLS_DIR = REPO_ROOT / "ops" / "devops" / "telemetry" +TELEMETRY_BUNDLE_PATH = REPO_ROOT / "out" / "telemetry" / "telemetry-offline-bundle.tar.gz" + +if str(RELEASE_TOOLS_DIR) not in sys.path: + sys.path.insert(0, str(RELEASE_TOOLS_DIR)) + +from verify_release import ( # type: ignore import-not-found + load_manifest, + resolve_path, + verify_release, +) + +import mirror_debug_store # type: ignore import-not-found + +DEFAULT_RELEASE_DIR = REPO_ROOT / "out" / "release" +DEFAULT_STAGING_DIR = REPO_ROOT / "out" / "offline-kit" / "staging" +DEFAULT_OUTPUT_DIR = REPO_ROOT / "out" / "offline-kit" / "dist" + +ARTIFACT_TARGETS = { + "sbom": Path("sboms"), + "provenance": Path("attest"), + "signature": Path("signatures"), + "metadata": Path("metadata/docker"), +} + + +class CommandError(RuntimeError): + """Raised when an external command fails.""" + + +def run(cmd: Iterable[str], *, cwd: Optional[Path] = None, env: Optional[Mapping[str, str]] = None) -> str: + process_env = dict(os.environ) + if env: + process_env.update(env) + result = subprocess.run( + list(cmd), + cwd=str(cwd) if cwd else None, + env=process_env, + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise CommandError( + f"Command failed ({result.returncode}): {' '.join(cmd)}\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + return result.stdout + + +def compute_sha256(path: Path) -> str: + sha = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(chunk) + return sha.hexdigest() + + +def utc_now_iso() -> str: + return dt.datetime.now(tz=dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def safe_component_name(name: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]", "-", name.strip().lower()) + + +def clean_directory(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def run_python_analyzer_smoke() -> None: + script = REPO_ROOT / "ops" / "offline-kit" / "run-python-analyzer-smoke.sh" + run(["bash", str(script)], cwd=REPO_ROOT) + + +def copy_if_exists(source: Path, target: Path) -> None: + if source.is_dir(): + shutil.copytree(source, target, dirs_exist_ok=True) + elif source.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def copy_release_manifests(release_dir: Path, staging_dir: Path) -> None: + manifest_dir = staging_dir / "manifest" + manifest_dir.mkdir(parents=True, exist_ok=True) + for name in ("release.yaml", "release.yaml.sha256", "release.json", "release.json.sha256"): + source = release_dir / name + if source.exists(): + shutil.copy2(source, manifest_dir / source.name) + + +def copy_component_artifacts( + manifest: Mapping[str, Any], + release_dir: Path, + staging_dir: Path, +) -> None: + components = manifest.get("components") or [] + for component in sorted(components, key=lambda entry: str(entry.get("name", ""))): + if not isinstance(component, Mapping): + continue + component_name = safe_component_name(str(component.get("name", "component"))) + for key, target_root in ARTIFACT_TARGETS.items(): + entry = component.get(key) + if not entry or not isinstance(entry, Mapping): + continue + path_str = entry.get("path") + if not path_str: + continue + resolved = resolve_path(str(path_str), release_dir) + if not resolved.exists(): + raise FileNotFoundError(f"Component '{component_name}' {key} artefact not found: {resolved}") + target_dir = staging_dir / target_root + target_dir.mkdir(parents=True, exist_ok=True) + target_name = f"{component_name}-{resolved.name}" if resolved.name else component_name + shutil.copy2(resolved, target_dir / target_name) + + +def copy_collections( + manifest: Mapping[str, Any], + release_dir: Path, + staging_dir: Path, +) -> None: + for collection, subdir in (("charts", Path("charts")), ("compose", Path("compose"))): + entries = manifest.get(collection) or [] + for entry in entries: + if not isinstance(entry, Mapping): + continue + path_str = entry.get("path") + if not path_str: + continue + resolved = resolve_path(str(path_str), release_dir) + if not resolved.exists(): + raise FileNotFoundError(f"{collection} artefact not found: {resolved}") + target_dir = staging_dir / subdir + target_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(resolved, target_dir / resolved.name) + + +def copy_debug_store(release_dir: Path, staging_dir: Path) -> None: + mirror_debug_store.main( + [ + "--release-dir", + str(release_dir), + "--offline-kit-dir", + str(staging_dir), + ] + ) + + +def copy_plugins_and_assets(staging_dir: Path) -> None: + copy_if_exists(REPO_ROOT / "plugins" / "scanner", staging_dir / "plugins" / "scanner") + copy_if_exists(REPO_ROOT / "certificates", staging_dir / "certificates") + copy_if_exists(REPO_ROOT / "seed-data", staging_dir / "seed-data") + docs_dir = staging_dir / "docs" + docs_dir.mkdir(parents=True, exist_ok=True) + copy_if_exists(REPO_ROOT / "docs" / "24_OFFLINE_KIT.md", docs_dir / "24_OFFLINE_KIT.md") + copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-collector.md", docs_dir / "telemetry-collector.md") + copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-storage.md", docs_dir / "telemetry-storage.md") + + +def package_telemetry_bundle(staging_dir: Path) -> None: + script = TELEMETRY_TOOLS_DIR / "package_offline_bundle.py" + if not script.exists(): + return + TELEMETRY_BUNDLE_PATH.parent.mkdir(parents=True, exist_ok=True) + run(["python", str(script), "--output", str(TELEMETRY_BUNDLE_PATH)], cwd=REPO_ROOT) + telemetry_dir = staging_dir / "telemetry" + telemetry_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(TELEMETRY_BUNDLE_PATH, telemetry_dir / TELEMETRY_BUNDLE_PATH.name) + sha_path = TELEMETRY_BUNDLE_PATH.with_suffix(TELEMETRY_BUNDLE_PATH.suffix + ".sha256") + if sha_path.exists(): + shutil.copy2(sha_path, telemetry_dir / sha_path.name) + + +def scan_files(staging_dir: Path, exclude: Optional[set[str]] = None) -> list[OrderedDict[str, Any]]: + entries: list[OrderedDict[str, Any]] = [] + exclude = exclude or set() + for path in sorted(staging_dir.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(staging_dir).as_posix() + if rel in exclude: + continue + entries.append( + OrderedDict( + ( + ("name", rel), + ("sha256", compute_sha256(path)), + ("size", path.stat().st_size), + ) + ) + ) + return entries + + +def write_offline_manifest( + staging_dir: Path, + version: str, + channel: str, + release_manifest_sha: Optional[str], +) -> tuple[Path, str]: + manifest_dir = staging_dir / "manifest" + manifest_dir.mkdir(parents=True, exist_ok=True) + offline_manifest_path = manifest_dir / "offline-manifest.json" + files = scan_files(staging_dir, exclude={"manifest/offline-manifest.json", "manifest/offline-manifest.json.sha256"}) + manifest_data = OrderedDict( + ( + ( + "bundle", + OrderedDict( + ( + ("version", version), + ("channel", channel), + ("capturedAt", utc_now_iso()), + ("releaseManifestSha256", release_manifest_sha), + ) + ), + ), + ("artifacts", files), + ) + ) + with offline_manifest_path.open("w", encoding="utf-8") as handle: + json.dump(manifest_data, handle, indent=2) + handle.write("\n") + manifest_sha = compute_sha256(offline_manifest_path) + (offline_manifest_path.with_suffix(".json.sha256")).write_text( + f"{manifest_sha} {offline_manifest_path.name}\n", + encoding="utf-8", + ) + return offline_manifest_path, manifest_sha + + +def tarinfo_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo: + tarinfo.uid = 0 + tarinfo.gid = 0 + tarinfo.uname = "" + tarinfo.gname = "" + tarinfo.mtime = 0 + return tarinfo + + +def create_tarball(staging_dir: Path, output_dir: Path, bundle_name: str) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + bundle_path = output_dir / f"{bundle_name}.tar.gz" + if bundle_path.exists(): + bundle_path.unlink() + with tarfile.open(bundle_path, "w:gz", compresslevel=9) as tar: + for path in sorted(staging_dir.rglob("*")): + if path.is_file(): + arcname = path.relative_to(staging_dir).as_posix() + tar.add(path, arcname=arcname, filter=tarinfo_filter) + return bundle_path + + +def sign_blob( + path: Path, + *, + key_ref: Optional[str], + identity_token: Optional[str], + password: Optional[str], + tlog_upload: bool, +) -> Optional[Path]: + if not key_ref and not identity_token: + return None + cmd = ["cosign", "sign-blob", "--yes", str(path)] + if key_ref: + cmd.extend(["--key", key_ref]) + if identity_token: + cmd.extend(["--identity-token", identity_token]) + if not tlog_upload: + cmd.append("--tlog-upload=false") + env = {"COSIGN_PASSWORD": password or ""} + signature = run(cmd, env=env) + sig_path = path.with_suffix(path.suffix + ".sig") + sig_path.write_text(signature, encoding="utf-8") + return sig_path + + +def build_offline_kit(args: argparse.Namespace) -> MutableMapping[str, Any]: + release_dir = args.release_dir.resolve() + staging_dir = args.staging_dir.resolve() + output_dir = args.output_dir.resolve() + + verify_release(release_dir) + if not args.skip_smoke: + run_python_analyzer_smoke() + clean_directory(staging_dir) + copy_debug_store(release_dir, staging_dir) + + manifest_data = load_manifest(release_dir) + release_manifest_sha = None + checksums = manifest_data.get("checksums") + if isinstance(checksums, Mapping): + release_manifest_sha = checksums.get("sha256") + + copy_release_manifests(release_dir, staging_dir) + copy_component_artifacts(manifest_data, release_dir, staging_dir) + copy_collections(manifest_data, release_dir, staging_dir) + copy_plugins_and_assets(staging_dir) + package_telemetry_bundle(staging_dir) + + offline_manifest_path, offline_manifest_sha = write_offline_manifest( + staging_dir, + args.version, + args.channel, + release_manifest_sha, + ) + bundle_name = f"stella-ops-offline-kit-{args.version}-{args.channel}" + bundle_path = create_tarball(staging_dir, output_dir, bundle_name) + bundle_sha = compute_sha256(bundle_path) + bundle_sha_prefixed = f"sha256:{bundle_sha}" + (bundle_path.with_suffix(".tar.gz.sha256")).write_text( + f"{bundle_sha} {bundle_path.name}\n", + encoding="utf-8", + ) + + signature_paths: dict[str, str] = {} + sig = sign_blob( + bundle_path, + key_ref=args.cosign_key, + identity_token=args.cosign_identity_token, + password=args.cosign_password, + tlog_upload=not args.no_transparency, + ) + if sig: + signature_paths["bundleSignature"] = str(sig) + manifest_sig = sign_blob( + offline_manifest_path, + key_ref=args.cosign_key, + identity_token=args.cosign_identity_token, + password=args.cosign_password, + tlog_upload=not args.no_transparency, + ) + if manifest_sig: + signature_paths["manifestSignature"] = str(manifest_sig) + + metadata = OrderedDict( + ( + ("bundleId", args.bundle_id or f"{args.version}-{args.channel}-{utc_now_iso()}"), + ("bundleName", bundle_path.name), + ("bundleSha256", bundle_sha_prefixed), + ("bundleSize", bundle_path.stat().st_size), + ("manifestName", offline_manifest_path.name), + ("manifestSha256", f"sha256:{offline_manifest_sha}"), + ("manifestSize", offline_manifest_path.stat().st_size), + ("channel", args.channel), + ("version", args.version), + ("capturedAt", utc_now_iso()), + ) + ) + + if sig: + metadata["bundleSignatureName"] = Path(sig).name + if manifest_sig: + metadata["manifestSignatureName"] = Path(manifest_sig).name + + metadata_path = output_dir / f"{bundle_name}.metadata.json" + with metadata_path.open("w", encoding="utf-8") as handle: + json.dump(metadata, handle, indent=2) + handle.write("\n") + + return OrderedDict( + ( + ("bundlePath", str(bundle_path)), + ("bundleSha256", bundle_sha), + ("manifestPath", str(offline_manifest_path)), + ("metadataPath", str(metadata_path)), + ("signatures", signature_paths), + ) + ) + + +def parse_args(argv: Optional[list[str]] = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True, help="Bundle version (e.g. 2025.10.0)") + parser.add_argument("--channel", default="edge", help="Release channel (default: %(default)s)") + parser.add_argument("--bundle-id", help="Optional explicit bundle identifier") + parser.add_argument( + "--release-dir", + type=Path, + default=DEFAULT_RELEASE_DIR, + help="Release artefact directory (default: %(default)s)", + ) + parser.add_argument( + "--staging-dir", + type=Path, + default=DEFAULT_STAGING_DIR, + help="Temporary staging directory (default: %(default)s)", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=DEFAULT_OUTPUT_DIR, + help="Destination directory for packaged bundles (default: %(default)s)", + ) + parser.add_argument("--cosign-key", dest="cosign_key", help="Cosign key reference for signing") + parser.add_argument("--cosign-password", dest="cosign_password", help="Cosign key password (if applicable)") + parser.add_argument("--cosign-identity-token", dest="cosign_identity_token", help="Cosign identity token") + parser.add_argument("--no-transparency", action="store_true", help="Disable Rekor transparency log uploads") + parser.add_argument("--skip-smoke", action="store_true", help="Skip analyzer smoke execution (testing only)") + return parser.parse_args(argv) + + +def main(argv: Optional[list[str]] = None) -> int: + args = parse_args(argv) + try: + result = build_offline_kit(args) + except Exception as exc: # pylint: disable=broad-except + print(f"offline-kit packaging failed: {exc}", file=sys.stderr) + return 1 + print("✅ Offline kit packaged") + for key, value in result.items(): + if isinstance(value, dict): + for sub_key, sub_val in value.items(): + print(f" - {key}.{sub_key}: {sub_val}") + else: + print(f" - {key}: {value}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/offline-kit/mirror_debug_store.py b/ops/offline-kit/mirror_debug_store.py new file mode 100644 index 00000000..334e40d9 --- /dev/null +++ b/ops/offline-kit/mirror_debug_store.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Mirror release debug-store artefacts into the Offline Kit staging tree. + +This helper copies the release `debug/` directory (including `.build-id/`, +`debug-manifest.json`, and the `.sha256` companion) into the Offline Kit +output directory and verifies the manifest hashes after the copy. A summary +document is written under `metadata/debug-store.json` so packaging jobs can +surface the available build-ids and validation status. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import pathlib +import shutil +import sys +from typing import Iterable, Tuple + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def compute_sha256(path: pathlib.Path) -> str: + import hashlib + + sha = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(chunk) + return sha.hexdigest() + + +def load_manifest(manifest_path: pathlib.Path) -> dict: + with manifest_path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def parse_manifest_sha(sha_path: pathlib.Path) -> str | None: + if not sha_path.exists(): + return None + text = sha_path.read_text(encoding="utf-8").strip() + if not text: + return None + # Allow either "" or " filename" formats. + return text.split()[0] + + +def iter_debug_files(base_dir: pathlib.Path) -> Iterable[pathlib.Path]: + for path in base_dir.rglob("*"): + if path.is_file(): + yield path + + +def copy_debug_store(source_root: pathlib.Path, target_root: pathlib.Path, *, dry_run: bool) -> None: + if dry_run: + print(f"[dry-run] Would copy '{source_root}' -> '{target_root}'") + return + + if target_root.exists(): + shutil.rmtree(target_root) + shutil.copytree(source_root, target_root) + + +def verify_debug_store(manifest: dict, offline_root: pathlib.Path) -> Tuple[int, int]: + """Return (verified_count, total_entries).""" + + artifacts = manifest.get("artifacts", []) + verified = 0 + for entry in artifacts: + debug_path = entry.get("debugPath") + expected_sha = entry.get("sha256") + expected_size = entry.get("size") + + if not debug_path or not expected_sha: + continue + + relative = pathlib.PurePosixPath(debug_path) + resolved = (offline_root.parent / relative).resolve() + + if not resolved.exists(): + raise FileNotFoundError(f"Debug artefact missing after mirror: {relative}") + + actual_sha = compute_sha256(resolved) + if actual_sha != expected_sha: + raise ValueError( + f"Digest mismatch for {relative}: expected {expected_sha}, found {actual_sha}" + ) + + if expected_size is not None: + actual_size = resolved.stat().st_size + if actual_size != expected_size: + raise ValueError( + f"Size mismatch for {relative}: expected {expected_size}, found {actual_size}" + ) + + verified += 1 + + return verified, len(artifacts) + + +def summarize_store(manifest: dict, manifest_sha: str | None, offline_root: pathlib.Path, summary_path: pathlib.Path) -> None: + debug_files = [ + path + for path in iter_debug_files(offline_root) + if path.suffix == ".debug" + ] + + total_size = sum(path.stat().st_size for path in debug_files) + build_ids = sorted( + {entry.get("buildId") for entry in manifest.get("artifacts", []) if entry.get("buildId")} + ) + + summary = { + "generatedAt": dt.datetime.now(tz=dt.timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z"), + "manifestGeneratedAt": manifest.get("generatedAt"), + "manifestSha256": manifest_sha, + "platforms": manifest.get("platforms") + or sorted({entry.get("platform") for entry in manifest.get("artifacts", []) if entry.get("platform")}), + "artifactCount": len(manifest.get("artifacts", [])), + "buildIds": { + "total": len(build_ids), + "samples": build_ids[:10], + }, + "debugFiles": { + "count": len(debug_files), + "totalSizeBytes": total_size, + }, + } + + summary_path.parent.mkdir(parents=True, exist_ok=True) + with summary_path.open("w", encoding="utf-8") as handle: + json.dump(summary, handle, indent=2) + handle.write("\n") + + +def resolve_release_debug_dir(base: pathlib.Path) -> pathlib.Path: + debug_dir = base / "debug" + if debug_dir.exists(): + return debug_dir + + # Allow specifying the channel directory directly (e.g. out/release/stable) + if base.name == "debug": + return base + + raise FileNotFoundError(f"Debug directory not found under '{base}'") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--release-dir", + type=pathlib.Path, + default=REPO_ROOT / "out" / "release", + help="Release output directory containing the debug store (default: %(default)s)", + ) + parser.add_argument( + "--offline-kit-dir", + type=pathlib.Path, + default=REPO_ROOT / "out" / "offline-kit", + help="Offline Kit staging directory (default: %(default)s)", + ) + parser.add_argument( + "--verify-only", + action="store_true", + help="Skip copying and only verify the existing offline kit debug store", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print actions without copying files", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + + try: + source_debug = resolve_release_debug_dir(args.release_dir.resolve()) + except FileNotFoundError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + target_root = (args.offline_kit_dir / "debug").resolve() + + if not args.verify_only: + copy_debug_store(source_debug, target_root, dry_run=args.dry_run) + if args.dry_run: + return 0 + + manifest_path = target_root / "debug-manifest.json" + if not manifest_path.exists(): + print(f"error: offline kit manifest missing at {manifest_path}", file=sys.stderr) + return 3 + + manifest = load_manifest(manifest_path) + manifest_sha_path = manifest_path.with_suffix(manifest_path.suffix + ".sha256") + recorded_sha = parse_manifest_sha(manifest_sha_path) + recomputed_sha = compute_sha256(manifest_path) + if recorded_sha and recorded_sha != recomputed_sha: + print( + f"warning: manifest SHA mismatch (recorded {recorded_sha}, recomputed {recomputed_sha}); updating checksum", + file=sys.stderr, + ) + manifest_sha_path.write_text(f"{recomputed_sha} {manifest_path.name}\n", encoding="utf-8") + + verified, total = verify_debug_store(manifest, target_root) + print(f"✔ verified {verified}/{total} debug artefacts (manifest SHA {recomputed_sha})") + + summary_path = args.offline_kit_dir / "metadata" / "debug-store.json" + summarize_store(manifest, recomputed_sha, target_root, summary_path) + print(f"ℹ summary written to {summary_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ops/offline-kit/run-python-analyzer-smoke.sh b/ops/offline-kit/run-python-analyzer-smoke.sh new file mode 100644 index 00000000..b4164c93 --- /dev/null +++ b/ops/offline-kit/run-python-analyzer-smoke.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(git -C "${BASH_SOURCE%/*}/.." rev-parse --show-toplevel 2>/dev/null || pwd)" +project_path="${repo_root}/src/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj" +output_dir="${repo_root}/out/analyzers/python" +plugin_dir="${repo_root}/plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python" + +to_win_path() { + if command -v wslpath >/dev/null 2>&1; then + wslpath -w "$1" + else + printf '%s\n' "$1" + fi +} + +rm -rf "${output_dir}" +project_path_win="$(to_win_path "$project_path")" +output_dir_win="$(to_win_path "$output_dir")" + +dotnet publish "$project_path_win" \ + --configuration Release \ + --output "$output_dir_win" \ + --self-contained false + +mkdir -p "${plugin_dir}" +cp "${output_dir}/StellaOps.Scanner.Analyzers.Lang.Python.dll" "${plugin_dir}/" +if [[ -f "${output_dir}/StellaOps.Scanner.Analyzers.Lang.Python.pdb" ]]; then + cp "${output_dir}/StellaOps.Scanner.Analyzers.Lang.Python.pdb" "${plugin_dir}/" +fi + +repo_root_win="$(to_win_path "$repo_root")" +exec dotnet run \ + --project "${repo_root_win}/tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj" \ + --configuration Release \ + -- --repo-root "${repo_root_win}" diff --git a/ops/offline-kit/test_build_offline_kit.py b/ops/offline-kit/test_build_offline_kit.py new file mode 100644 index 00000000..298737c0 --- /dev/null +++ b/ops/offline-kit/test_build_offline_kit.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import json +import tarfile +import tempfile +import unittest +import argparse +import sys +from collections import OrderedDict +from pathlib import Path + +sys.path.append(str(Path(__file__).resolve().parent)) + +from build_release import write_manifest # type: ignore import-not-found + +from build_offline_kit import build_offline_kit, compute_sha256 # type: ignore import-not-found + + +class OfflineKitBuilderTests(unittest.TestCase): + def setUp(self) -> None: + self._temp = tempfile.TemporaryDirectory() + self.base_path = Path(self._temp.name) + self.out_dir = self.base_path / "out" + self.release_dir = self.out_dir / "release" + self.staging_dir = self.base_path / "staging" + self.output_dir = self.base_path / "dist" + self._create_sample_release() + + def tearDown(self) -> None: + self._temp.cleanup() + + def _relative_to_out(self, path: Path) -> str: + return path.relative_to(self.out_dir).as_posix() + + def _write_json(self, path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2) + handle.write("\n") + + def _create_sample_release(self) -> None: + self.release_dir.mkdir(parents=True, exist_ok=True) + + sbom_path = self.release_dir / "artifacts/sboms/sample.cyclonedx.json" + sbom_path.parent.mkdir(parents=True, exist_ok=True) + sbom_path.write_text('{"bomFormat":"CycloneDX","specVersion":"1.5"}\n', encoding="utf-8") + sbom_sha = compute_sha256(sbom_path) + + provenance_path = self.release_dir / "artifacts/provenance/sample.provenance.json" + self._write_json( + provenance_path, + { + "buildDefinition": {"buildType": "https://example/build"}, + "runDetails": {"builder": {"id": "https://example/ci"}}, + }, + ) + provenance_sha = compute_sha256(provenance_path) + + signature_path = self.release_dir / "artifacts/signatures/sample.signature" + signature_path.parent.mkdir(parents=True, exist_ok=True) + signature_path.write_text("signature-data\n", encoding="utf-8") + signature_sha = compute_sha256(signature_path) + + metadata_path = self.release_dir / "artifacts/metadata/sample.metadata.json" + self._write_json(metadata_path, {"digest": "sha256:1234"}) + metadata_sha = compute_sha256(metadata_path) + + chart_path = self.release_dir / "helm/stellaops-1.0.0.tgz" + chart_path.parent.mkdir(parents=True, exist_ok=True) + chart_path.write_bytes(b"helm-chart-data") + chart_sha = compute_sha256(chart_path) + + compose_path = self.release_dir.parent / "deploy/compose/docker-compose.dev.yaml" + compose_path.parent.mkdir(parents=True, exist_ok=True) + compose_path.write_text("services: {}\n", encoding="utf-8") + compose_sha = compute_sha256(compose_path) + + debug_file = self.release_dir / "debug/.build-id/ab/cdef.debug" + debug_file.parent.mkdir(parents=True, exist_ok=True) + debug_file.write_bytes(b"\x7fELFDEBUGDATA") + debug_sha = compute_sha256(debug_file) + + debug_manifest_path = self.release_dir / "debug/debug-manifest.json" + debug_manifest = OrderedDict( + ( + ("generatedAt", "2025-10-26T00:00:00Z"), + ("version", "1.0.0"), + ("channel", "edge"), + ( + "artifacts", + [ + OrderedDict( + ( + ("buildId", "abcdef1234"), + ("platform", "linux/amd64"), + ("debugPath", "debug/.build-id/ab/cdef.debug"), + ("sha256", debug_sha), + ("size", debug_file.stat().st_size), + ("components", ["sample"]), + ("images", ["registry.example/sample@sha256:feedface"]), + ("sources", ["app/sample.dll"]), + ) + ) + ], + ), + ) + ) + self._write_json(debug_manifest_path, debug_manifest) + debug_manifest_sha = compute_sha256(debug_manifest_path) + (debug_manifest_path.with_suffix(debug_manifest_path.suffix + ".sha256")).write_text( + f"{debug_manifest_sha} {debug_manifest_path.name}\n", + encoding="utf-8", + ) + + manifest = OrderedDict( + ( + ( + "release", + OrderedDict( + ( + ("version", "1.0.0"), + ("channel", "edge"), + ("date", "2025-10-26T00:00:00Z"), + ("calendar", "2025.10"), + ) + ), + ), + ( + "components", + [ + OrderedDict( + ( + ("name", "sample"), + ("image", "registry.example/sample@sha256:feedface"), + ("tags", ["registry.example/sample:1.0.0"]), + ( + "sbom", + OrderedDict( + ( + ("path", self._relative_to_out(sbom_path)), + ("sha256", sbom_sha), + ) + ), + ), + ( + "provenance", + OrderedDict( + ( + ("path", self._relative_to_out(provenance_path)), + ("sha256", provenance_sha), + ) + ), + ), + ( + "signature", + OrderedDict( + ( + ("path", self._relative_to_out(signature_path)), + ("sha256", signature_sha), + ("ref", "sigstore://example"), + ("tlogUploaded", True), + ) + ), + ), + ( + "metadata", + OrderedDict( + ( + ("path", self._relative_to_out(metadata_path)), + ("sha256", metadata_sha), + ) + ), + ), + ) + ) + ], + ), + ( + "charts", + [ + OrderedDict( + ( + ("name", "stellaops"), + ("version", "1.0.0"), + ("path", self._relative_to_out(chart_path)), + ("sha256", chart_sha), + ) + ) + ], + ), + ( + "compose", + [ + OrderedDict( + ( + ("name", "docker-compose.dev.yaml"), + ("path", compose_path.relative_to(self.out_dir).as_posix()), + ("sha256", compose_sha), + ) + ) + ], + ), + ( + "debugStore", + OrderedDict( + ( + ("manifest", "debug/debug-manifest.json"), + ("sha256", debug_manifest_sha), + ("entries", 1), + ("platforms", ["linux/amd64"]), + ("directory", "debug/.build-id"), + ) + ), + ), + ) + ) + write_manifest(manifest, self.release_dir) + + def test_build_offline_kit(self) -> None: + args = argparse.Namespace( + version="2025.10.0", + channel="edge", + bundle_id="bundle-001", + release_dir=self.release_dir, + staging_dir=self.staging_dir, + output_dir=self.output_dir, + cosign_key=None, + cosign_password=None, + cosign_identity_token=None, + no_transparency=False, + skip_smoke=True, + ) + result = build_offline_kit(args) + bundle_path = Path(result["bundlePath"]) + self.assertTrue(bundle_path.exists()) + offline_manifest = self.output_dir.parent / "staging" / "manifest" / "offline-manifest.json" + self.assertTrue(offline_manifest.exists()) + + with offline_manifest.open("r", encoding="utf-8") as handle: + manifest_data = json.load(handle) + artifacts = manifest_data["artifacts"] + self.assertTrue(any(item["name"].startswith("sboms/") for item in artifacts)) + + metadata_path = Path(result["metadataPath"]) + data = json.loads(metadata_path.read_text(encoding="utf-8")) + self.assertTrue(data["bundleSha256"].startswith("sha256:")) + self.assertTrue(data["manifestSha256"].startswith("sha256:")) + + with tarfile.open(bundle_path, "r:gz") as tar: + members = tar.getnames() + self.assertIn("manifest/release.yaml", members) + self.assertTrue(any(name.startswith("sboms/sample-") for name in members)) + + +if __name__ == "__main__": + unittest.main() diff --git a/out/analyzers/python/StellaOps.Auth.Abstractions.xml b/out/analyzers/python/StellaOps.Auth.Abstractions.xml new file mode 100644 index 00000000..5b77e659 --- /dev/null +++ b/out/analyzers/python/StellaOps.Auth.Abstractions.xml @@ -0,0 +1,422 @@ + + + + StellaOps.Auth.Abstractions + + + + + Canonical telemetry metadata for the StellaOps Authority stack. + + + + + service.name resource attribute recorded by Authority components. + + + + + service.namespace resource attribute aligning Authority with other StellaOps services. + + + + + Activity source identifier used by Authority instrumentation. + + + + + Meter name used by Authority instrumentation. + + + + + Builds the default set of resource attributes (service name/namespace/version). + + Optional assembly used to resolve the service version. + + + + Resolves the service version string from the provided assembly (defaults to the Authority telemetry assembly). + + + + + Represents an IP network expressed in CIDR notation. + + + + + Initialises a new . + + Canonical network address with host bits zeroed. + Prefix length (0-32 for IPv4, 0-128 for IPv6). + + + + Canonical network address with host bits zeroed. + + + + + Prefix length. + + + + + Attempts to parse the supplied value as CIDR notation or a single IP address. + + Thrown when the input is not recognised. + + + + Attempts to parse the supplied value as CIDR notation or a single IP address. + + + + + Determines whether the provided address belongs to this network. + + + + + + + + Evaluates remote addresses against configured network masks. + + + + + Creates a matcher from raw CIDR strings. + + Sequence of CIDR entries or IP addresses. + Thrown when a value cannot be parsed. + + + + Creates a matcher from already parsed masks. + + Sequence of network masks. + + + + Gets a matcher that allows every address. + + + + + Gets a matcher that denies every address (no masks configured). + + + + + Indicates whether this matcher has no masks configured and does not allow all. + + + + + Returns the configured masks. + + + + + Checks whether the provided address matches any of the configured masks. + + Remote address to test. + true when the address is allowed. + + + + Default authentication constants used by StellaOps resource servers and clients. + + + + + Default authentication scheme for StellaOps bearer tokens. + + + + + Logical authentication type attached to . + + + + + Policy prefix applied to named authorization policies. + + + + + Canonical claim type identifiers used across StellaOps services. + + + + + Subject identifier claim (maps to sub in JWTs). + + + + + StellaOps tenant identifier claim (multi-tenant deployments). + + + + + OAuth2/OIDC client identifier claim (maps to client_id). + + + + + Unique token identifier claim (maps to jti). + + + + + Authentication method reference claim (amr). + + + + + Space separated scope list (scope). + + + + + Individual scope items (scp). + + + + + OAuth2 resource audiences (aud). + + + + + Identity provider hint for downstream services. + + + + + Session identifier claim (sid). + + + + + Fluent helper used to construct instances that follow StellaOps conventions. + + + + + Adds or replaces the canonical subject identifier. + + + + + Adds or replaces the canonical client identifier. + + + + + Adds or replaces the tenant identifier claim. + + + + + Adds or replaces the user display name claim. + + + + + Adds or replaces the identity provider claim. + + + + + Adds or replaces the session identifier claim. + + + + + Adds or replaces the token identifier claim. + + + + + Adds or replaces the authentication method reference claim. + + + + + Sets the name claim type appended when building the . + + + + + Sets the role claim type appended when building the . + + + + + Sets the authentication type stamped on the . + + + + + Registers the supplied scopes (normalised to lower-case, deduplicated, sorted). + + + + + Registers the supplied audiences (trimmed, deduplicated, sorted). + + + + + Adds a single audience. + + + + + Adds an arbitrary claim (no deduplication is performed). + + + + + Adds multiple claims (incoming claims are cloned to enforce value trimming). + + + + + Adds an iat (issued at) claim using Unix time seconds. + + + + + Adds an nbf (not before) claim using Unix time seconds. + + + + + Adds an exp (expires) claim using Unix time seconds. + + + + + Returns the normalised scope list (deduplicated + sorted). + + + + + Returns the normalised audience list (deduplicated + sorted). + + + + + Builds the immutable instance based on the registered data. + + + + + Factory helpers for returning RFC 7807 problem responses using StellaOps conventions. + + + + + Produces a 401 problem response indicating authentication is required. + + + + + Produces a 401 problem response for invalid, expired, or revoked tokens. + + + + + Produces a 403 problem response when access is denied. + + + + + Produces a 403 problem response for insufficient scopes. + + + + + Canonical scope names supported by StellaOps services. + + + + + Scope required to trigger Concelier jobs. + + + + + Scope required to manage Concelier merge operations. + + + + + Scope granting administrative access to Authority user management. + + + + + Scope granting administrative access to Authority client registrations. + + + + + Scope granting read-only access to Authority audit logs. + + + + + Synthetic scope representing trusted network bypass. + + + + + Scope granting read-only access to raw advisory ingestion data. + + + + + Scope granting write access for raw advisory ingestion. + + + + + Scope granting read-only access to raw VEX ingestion data. + + + + + Scope granting write access for raw VEX ingestion. + + + + + Scope granting permission to execute aggregation-only contract verification. + + + + + Normalises a scope string (trim/convert to lower case). + + Scope raw value. + Normalised scope or null when the input is blank. + + + + Checks whether the provided scope is registered as a built-in StellaOps scope. + + + + + Returns the full set of built-in scopes. + + + + diff --git a/out/analyzers/python/StellaOps.Auth.Client.xml b/out/analyzers/python/StellaOps.Auth.Client.xml new file mode 100644 index 00000000..3a8a3b98 --- /dev/null +++ b/out/analyzers/python/StellaOps.Auth.Client.xml @@ -0,0 +1,233 @@ + + + + StellaOps.Auth.Client + + + + + File-based token cache suitable for CLI/offline usage. + + + + + In-memory token cache suitable for service scenarios. + + + + + Abstraction for caching StellaOps tokens. + + + + + Retrieves a cached token entry, if present. + + + + + Stores or updates a token entry for the specified key. + + + + + Removes the cached entry for the specified key. + + + + + Abstraction for requesting tokens from StellaOps Authority. + + + + + Requests an access token using the resource owner password credentials flow. + + + + + Requests an access token using the client credentials flow. + + + + + Retrieves the cached JWKS document. + + + + + Retrieves a cached token entry. + + + + + Persists a token entry in the cache. + + + + + Removes a cached entry. + + + + + DI helpers for the StellaOps auth client. + + + + + Registers the StellaOps auth client with the provided configuration. + + + + + Registers a file-backed token cache implementation. + + + + + Options controlling the StellaOps authentication client. + + + + + Authority (issuer) base URL. + + + + + OAuth client identifier (optional for password flow). + + + + + OAuth client secret (optional for public clients). + + + + + Default scopes requested for flows that do not explicitly override them. + + + + + Retry delays applied by HTTP retry policy (empty uses defaults). + + + + + Gets or sets a value indicating whether HTTP retry policies are enabled. + + + + + Timeout applied to discovery and token HTTP requests. + + + + + Lifetime of cached discovery metadata. + + + + + Lifetime of cached JWKS metadata. + + + + + Buffer applied when determining cache expiration (default: 30 seconds). + + + + + Gets or sets a value indicating whether cached discovery/JWKS responses may be served when the Authority is unreachable. + + + + + Additional tolerance window during which stale cache entries remain valid if offline fallback is allowed. + + + + + Parsed Authority URI (populated after validation). + + + + + Normalised scope list (populated after validation). + + + + + Normalised retry delays (populated after validation). + + + + + Validates required values and normalises scope entries. + + + + + Caches Authority discovery metadata. + + + + + Minimal OpenID Connect configuration representation. + + + + + Minimal OpenID Connect configuration representation. + + + + + Caches JWKS documents for Authority. + + + + + Represents a cached token entry. + + + + + Represents a cached token entry. + + + + + Determines whether the token is expired given the provided . + + + + + Creates a copy with scopes normalised. + + + + + Default implementation of . + + + + + Represents an issued token with metadata. + + + + + Represents an issued token with metadata. + + + + + Converts the result to a cache entry. + + + + diff --git a/out/analyzers/python/StellaOps.Scanner.Analyzers.Lang.Python.deps.json b/out/analyzers/python/StellaOps.Scanner.Analyzers.Lang.Python.deps.json new file mode 100644 index 00000000..1e6bd61b --- /dev/null +++ b/out/analyzers/python/StellaOps.Scanner.Analyzers.Lang.Python.deps.json @@ -0,0 +1,858 @@ +{ + "runtimeTarget": { + "name": ".NETCoreApp,Version=v10.0", + "signature": "" + }, + "compilationOptions": {}, + "targets": { + ".NETCoreApp,Version=v10.0": { + "StellaOps.Scanner.Analyzers.Lang.Python/1.0.0": { + "dependencies": { + "SharpCompress": "0.41.0", + "StellaOps.Scanner.Analyzers.Lang": "1.0.0" + }, + "runtime": { + "StellaOps.Scanner.Analyzers.Lang.Python.dll": {} + } + }, + "Konscious.Security.Cryptography.Argon2/1.3.1": { + "dependencies": { + "Konscious.Security.Cryptography.Blake2": "1.1.1" + }, + "runtime": { + "lib/net8.0/Konscious.Security.Cryptography.Argon2.dll": { + "assemblyVersion": "1.3.1.0", + "fileVersion": "1.3.1.0" + } + } + }, + "Konscious.Security.Cryptography.Blake2/1.1.1": { + "runtime": { + "lib/net8.0/Konscious.Security.Cryptography.Blake2.dll": { + "assemblyVersion": "1.1.1.0", + "fileVersion": "1.1.1.0" + } + } + }, + "Microsoft.Extensions.Configuration/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Configuration.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Configuration.Abstractions/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Configuration.Abstractions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Configuration.Binder/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Configuration.Binder.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Configuration.EnvironmentVariables/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Configuration.EnvironmentVariables.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Configuration.FileExtensions/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.FileProviders.Physical": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Configuration.FileExtensions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Configuration.Json/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Configuration.Json.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.DependencyInjection/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.DependencyInjection.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions/10.0.0-rc.2.25502.107": { + "runtime": { + "lib/net10.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Diagnostics/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Diagnostics.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Diagnostics.Abstractions/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Options": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Diagnostics.Abstractions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.FileProviders.Abstractions/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.FileProviders.Abstractions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.FileProviders.Physical/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.FileSystemGlobbing": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.FileProviders.Physical.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.FileSystemGlobbing/10.0.0-rc.2.25502.107": { + "runtime": { + "lib/net10.0/Microsoft.Extensions.FileSystemGlobbing.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Http/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Diagnostics": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Logging": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Options": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Http.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Http.Polly/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Http": "10.0.0-rc.2.25502.107", + "Polly": "7.2.4", + "Polly.Extensions.Http": "3.0.0" + }, + "runtime": { + "lib/netstandard2.0/Microsoft.Extensions.Http.Polly.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Logging/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Options": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Logging.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Logging.Abstractions/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Logging.Abstractions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Options/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Options.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions/10.0.0-rc.2.25502.107": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Binder": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Options": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Primitives": "10.0.0-rc.2.25502.107" + }, + "runtime": { + "lib/net10.0/Microsoft.Extensions.Options.ConfigurationExtensions.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.Extensions.Primitives/10.0.0-rc.2.25502.107": { + "runtime": { + "lib/net10.0/Microsoft.Extensions.Primitives.dll": { + "assemblyVersion": "10.0.0.0", + "fileVersion": "10.0.25.50307" + } + } + }, + "Microsoft.IdentityModel.Abstractions/8.14.0": { + "runtime": { + "lib/net9.0/Microsoft.IdentityModel.Abstractions.dll": { + "assemblyVersion": "8.14.0.0", + "fileVersion": "8.14.0.60815" + } + } + }, + "Microsoft.IdentityModel.JsonWebTokens/7.2.0": { + "dependencies": { + "Microsoft.IdentityModel.Tokens": "8.14.0" + }, + "runtime": { + "lib/net8.0/Microsoft.IdentityModel.JsonWebTokens.dll": { + "assemblyVersion": "7.2.0.0", + "fileVersion": "7.2.0.50110" + } + } + }, + "Microsoft.IdentityModel.Logging/8.14.0": { + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.14.0" + }, + "runtime": { + "lib/net9.0/Microsoft.IdentityModel.Logging.dll": { + "assemblyVersion": "8.14.0.0", + "fileVersion": "8.14.0.60815" + } + } + }, + "Microsoft.IdentityModel.Tokens/8.14.0": { + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.IdentityModel.Logging": "8.14.0" + }, + "runtime": { + "lib/net9.0/Microsoft.IdentityModel.Tokens.dll": { + "assemblyVersion": "8.14.0.0", + "fileVersion": "8.14.0.60815" + } + } + }, + "NetEscapades.Configuration.Yaml/2.1.0": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.0-rc.2.25502.107", + "YamlDotNet": "9.1.0" + }, + "runtime": { + "lib/netstandard2.0/NetEscapades.Configuration.Yaml.dll": { + "assemblyVersion": "2.1.0.0", + "fileVersion": "2.1.0.0" + } + } + }, + "Pipelines.Sockets.Unofficial/2.2.8": { + "runtime": { + "lib/net5.0/Pipelines.Sockets.Unofficial.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "2.2.8.1080" + } + } + }, + "Polly/7.2.4": { + "runtime": { + "lib/netstandard2.0/Polly.dll": { + "assemblyVersion": "7.0.0.0", + "fileVersion": "7.2.4.982" + } + } + }, + "Polly.Extensions.Http/3.0.0": { + "dependencies": { + "Polly": "7.2.4" + }, + "runtime": { + "lib/netstandard2.0/Polly.Extensions.Http.dll": { + "assemblyVersion": "3.0.0.0", + "fileVersion": "3.0.0.0" + } + } + }, + "SharpCompress/0.41.0": { + "dependencies": { + "ZstdSharp.Port": "0.8.6" + }, + "runtime": { + "lib/net8.0/SharpCompress.dll": { + "assemblyVersion": "0.41.0.0", + "fileVersion": "0.41.0.0" + } + } + }, + "StackExchange.Redis/2.8.24": { + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "Pipelines.Sockets.Unofficial": "2.2.8" + }, + "runtime": { + "lib/net8.0/StackExchange.Redis.dll": { + "assemblyVersion": "2.0.0.0", + "fileVersion": "2.8.24.3255" + } + } + }, + "System.IdentityModel.Tokens.Jwt/7.2.0": { + "dependencies": { + "Microsoft.IdentityModel.JsonWebTokens": "7.2.0", + "Microsoft.IdentityModel.Tokens": "8.14.0" + }, + "runtime": { + "lib/net8.0/System.IdentityModel.Tokens.Jwt.dll": { + "assemblyVersion": "7.2.0.0", + "fileVersion": "7.2.0.50110" + } + } + }, + "YamlDotNet/9.1.0": { + "runtime": { + "lib/netstandard2.1/YamlDotNet.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.1.0.0" + } + } + }, + "ZstdSharp.Port/0.8.6": { + "runtime": { + "lib/net9.0/ZstdSharp.dll": { + "assemblyVersion": "0.8.6.0", + "fileVersion": "0.8.6.0" + } + } + }, + "StellaOps.Auth.Abstractions/1.0.0-preview.1": { + "dependencies": { + "SharpCompress": "0.41.0" + }, + "runtime": { + "StellaOps.Auth.Abstractions.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Auth.Client/1.0.0-preview.1": { + "dependencies": { + "Microsoft.Extensions.Http.Polly": "10.0.0-rc.2.25502.107", + "Microsoft.IdentityModel.Tokens": "8.14.0", + "SharpCompress": "0.41.0", + "StellaOps.Auth.Abstractions": "1.0.0-preview.1", + "StellaOps.Configuration": "1.0.0" + }, + "runtime": { + "StellaOps.Auth.Client.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Auth.Security/1.0.0-preview.1": { + "dependencies": { + "Microsoft.IdentityModel.Tokens": "8.14.0", + "SharpCompress": "0.41.0", + "StackExchange.Redis": "2.8.24", + "System.IdentityModel.Tokens.Jwt": "7.2.0" + }, + "runtime": { + "StellaOps.Auth.Security.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Authority.Plugins.Abstractions/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "SharpCompress": "0.41.0", + "StellaOps.Auth.Abstractions": "1.0.0-preview.1", + "StellaOps.Cryptography": "1.0.0" + }, + "runtime": { + "StellaOps.Authority.Plugins.Abstractions.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Configuration/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Binder": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Configuration.Json": "10.0.0-rc.2.25502.107", + "NetEscapades.Configuration.Yaml": "2.1.0", + "SharpCompress": "0.41.0", + "StellaOps.Authority.Plugins.Abstractions": "1.0.0", + "StellaOps.Cryptography": "1.0.0" + }, + "runtime": { + "StellaOps.Configuration.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Cryptography/1.0.0": { + "dependencies": { + "Konscious.Security.Cryptography.Argon2": "1.3.1", + "Microsoft.IdentityModel.Tokens": "8.14.0", + "SharpCompress": "0.41.0" + }, + "runtime": { + "StellaOps.Cryptography.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.DependencyInjection/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "SharpCompress": "0.41.0" + }, + "runtime": { + "StellaOps.DependencyInjection.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Plugin/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "SharpCompress": "0.41.0", + "StellaOps.DependencyInjection": "1.0.0" + }, + "runtime": { + "StellaOps.Plugin.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Scanner.Analyzers.Lang/1.0.0": { + "dependencies": { + "SharpCompress": "0.41.0", + "StellaOps.Plugin": "1.0.0", + "StellaOps.Scanner.Core": "1.0.0" + }, + "runtime": { + "StellaOps.Scanner.Analyzers.Lang.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Scanner.Core/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "10.0.0-rc.2.25502.107", + "Microsoft.Extensions.Options": "10.0.0-rc.2.25502.107", + "SharpCompress": "0.41.0", + "StellaOps.Auth.Client": "1.0.0-preview.1", + "StellaOps.Auth.Security": "1.0.0-preview.1" + }, + "runtime": { + "StellaOps.Scanner.Core.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + } + } + }, + "libraries": { + "StellaOps.Scanner.Analyzers.Lang.Python/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "Konscious.Security.Cryptography.Argon2/1.3.1": { + "type": "package", + "serviceable": true, + "sha512": "sha512-T+OAGwzYYXftahpOxO7J4xA5K6urxwGnWQf3M+Jpi+76Azv/0T3M5SuN+h7/QvXuiqNw3ZEZ5QqVLI5ygDAylw==", + "path": "konscious.security.cryptography.argon2/1.3.1", + "hashPath": "konscious.security.cryptography.argon2.1.3.1.nupkg.sha512" + }, + "Konscious.Security.Cryptography.Blake2/1.1.1": { + "type": "package", + "serviceable": true, + "sha512": "sha512-odwOyzj/J/lHJZNwFWJGU/LRecBShupAJ2S8TQqZfhUe9niHzu/voBYK5wuVKsvSpzbfupKQYZguVyIk1sgOkQ==", + "path": "konscious.security.cryptography.blake2/1.1.1", + "hashPath": "konscious.security.cryptography.blake2.1.1.1.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-273Ggibh3DdVrj47ENbUGIirOiqmLTAizpkvOD584Ps6NL/CMXPzesijnJgsjp7Fv/UCp69FKYBaSxZZ3q5R9g==", + "path": "microsoft.extensions.configuration/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.configuration.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration.Abstractions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-H+i/Qy30Rg/K9BcW2Z6DCHPCzwMH3bCwNOjEz31shWTUDK8GeeeMnrKVusprTcRA2Y6yPST+hg2zc3whPEs14Q==", + "path": "microsoft.extensions.configuration.abstractions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.configuration.abstractions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration.Binder/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-aA6/V6lw1Gueyb1PqhHAl/i/qUUuv+Fusfk4oaMOzzOjspBkYtPpNHCmml/0t1x0/DnZoed+u2WwpP+mSwd8Dg==", + "path": "microsoft.extensions.configuration.binder/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.configuration.binder.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration.EnvironmentVariables/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-2SV60IUAWfluZv2YHNZ+nUOljYHGIsy96FpJs+N9/bgKDYs9qr6DdzPeIhiHrz+XvRzbybvcwtTBf5dKrYN4oA==", + "path": "microsoft.extensions.configuration.environmentvariables/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.configuration.environmentvariables.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration.FileExtensions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-5KrgXSTFR8cFLmDXXoT7GLVvDyHNw0Z9xG4doD78Q/HdlAR4jiMzmLLS9GFXrPGopmC6qqEZr2VBJHEu16INcA==", + "path": "microsoft.extensions.configuration.fileextensions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.configuration.fileextensions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration.Json/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-USwHuFz4BFKoaqSydHWH/d7Mr+fVsAh9S0S9pdsdHro1IixMbqQ9Gpo2sEZf25e3tZSq/ts6XsVmrQWmxmDhYA==", + "path": "microsoft.extensions.configuration.json/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.configuration.json.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.DependencyInjection/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-mDw80K98jBWCyLFCra51PRv+Ttnjse1lZIzXEFybKby0/ajBFTEeHj/4r/QJexmb8Uun0yaFH1HlFtmHP1YEVA==", + "path": "microsoft.extensions.dependencyinjection/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.dependencyinjection.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.DependencyInjection.Abstractions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-8jujunpkNNfTkE9PFHp9/aD6GPKVfNCuz8tUbzOcyU5tQOCoIZId4hwQNVx3Tb8XEWw9BYdh0k5vPpqdCM+UtA==", + "path": "microsoft.extensions.dependencyinjection.abstractions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.dependencyinjection.abstractions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Diagnostics/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-tQfQFXI+ZQcL2RzDarDLx3Amh0WCp1KPGp1ie3y/CMV5hDhEq98WTmcMoXrFY0GkYLEaCQlVi2A6qVLcooG2Ow==", + "path": "microsoft.extensions.diagnostics/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.diagnostics.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Diagnostics.Abstractions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-x6XVv3RiwOlN2unjyX/Zat0gI0HiRoDDdjkwBCwsMftYWpbJu4SiyRwDbrv2zAF8v8nbEEvcWi3/pUxZfaqLQw==", + "path": "microsoft.extensions.diagnostics.abstractions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.diagnostics.abstractions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.FileProviders.Abstractions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-dOpmW14MkOZIwV6269iXhoMp6alCHBoxqCR4pJ37GLjFaBIyzsIy+Ra8tsGmjHtFvEHKq0JRDIsb1PUkrK+yxw==", + "path": "microsoft.extensions.fileproviders.abstractions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.fileproviders.abstractions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.FileProviders.Physical/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-3+RiR6TEakDL0dCUqR7PjFffyrVMLdx/vAVBiN1mGmwScKYCTePIkYVkWsX85CTKh7R9J4M9C1MHzVdjbKcg3g==", + "path": "microsoft.extensions.fileproviders.physical/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.fileproviders.physical.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.FileSystemGlobbing/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-XtcPOKB7sMFzj8SxaOglZV3eaqZ1GxUMVZTwaz4pRpBt0S45ghb836uUej4YaI8EzsnUJoqzOIKrTW4CDJMfVw==", + "path": "microsoft.extensions.filesystemglobbing/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.filesystemglobbing.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Http/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-d60bvi/NpzkpVlSpxZqOfdjX1hrQgL/byWVc3PryjbmB7zvfLtqQbYifjEWToqtS0Fb1rGnkuVI5JEdOnK1tNQ==", + "path": "microsoft.extensions.http/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.http.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Http.Polly/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-aY5vLcrhdXCHsCjYI2lNwfat2vdSuiPs0FFZiy7IM6zcyqdxaefG8J8ezTKkZyiuAtznjVJJT70B660l/WlsxA==", + "path": "microsoft.extensions.http.polly/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.http.polly.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Logging/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-q2C5gq86qkTmcYSJJSnw8sgTUyuqENYSOjk/NOYjHnYlKSrK3oI9Rjv1bWFpx2I3Btq9ZBEJb9aMM+IUQ0PvZA==", + "path": "microsoft.extensions.logging/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.logging.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Logging.Abstractions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-SKKKZjyCpBaDQ7yuFjdk6ELnRBRWeZsbnzUfo59Wc4PGhgf92chE3we/QlT6nk6NqlWcUgH/jogM+B/uq/Qdnw==", + "path": "microsoft.extensions.logging.abstractions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.logging.abstractions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Options/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-Ib6BCCjisp7ZUdhtNpSulFO0ODhz/IE4ZZd8OCqQWoRs363BQ0QOZi9KwpqpiEWo51S0kIXWqNicDPGXwpt9pQ==", + "path": "microsoft.extensions.options/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.options.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Options.ConfigurationExtensions/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-MFbT8+JKX49YCXEFvlZDzQzI/R3QKzRZlb4dSud+569cMgA9hWbndjWWvOgGASoRcXynGRrBSq1Bw3PeCsB5/Q==", + "path": "microsoft.extensions.options.configurationextensions/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.options.configurationextensions.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.Extensions.Primitives/10.0.0-rc.2.25502.107": { + "type": "package", + "serviceable": true, + "sha512": "sha512-9pm2zqqn5u/OsKs2zgkhJEQQeMx9KkVOWPdHrs7Kt5sfpk+eIh/gmpi/mMH/ljS2T/PFsFdCEtm+GS/6l7zoZA==", + "path": "microsoft.extensions.primitives/10.0.0-rc.2.25502.107", + "hashPath": "microsoft.extensions.primitives.10.0.0-rc.2.25502.107.nupkg.sha512" + }, + "Microsoft.IdentityModel.Abstractions/8.14.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-iwbCpSjD3ehfTwBhtSNEtKPK0ICun6ov7Ibx6ISNA9bfwIyzI2Siwyi9eJFCJBwxowK9xcA1mj+jBWiigeqgcQ==", + "path": "microsoft.identitymodel.abstractions/8.14.0", + "hashPath": "microsoft.identitymodel.abstractions.8.14.0.nupkg.sha512" + }, + "Microsoft.IdentityModel.JsonWebTokens/7.2.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-zLFA9IBxDWw6Y1nz2PPZyQvF+ZZ4aW1pwgtwusQB39lgxOc2xVqZ8gitsuT1rwyuIbchGOWbax4fsJ8OgGRxSQ==", + "path": "microsoft.identitymodel.jsonwebtokens/7.2.0", + "hashPath": "microsoft.identitymodel.jsonwebtokens.7.2.0.nupkg.sha512" + }, + "Microsoft.IdentityModel.Logging/8.14.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-eqqnemdW38CKZEHS6diA50BV94QICozDZEvSrsvN3SJXUFwVB9gy+/oz76gldP7nZliA16IglXjXTCTdmU/Ejg==", + "path": "microsoft.identitymodel.logging/8.14.0", + "hashPath": "microsoft.identitymodel.logging.8.14.0.nupkg.sha512" + }, + "Microsoft.IdentityModel.Tokens/8.14.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-lKIZiBiGd36k02TCdMHp1KlNWisyIvQxcYJvIkz7P4gSQ9zi8dgh6S5Grj8NNG7HWYIPfQymGyoZ6JB5d1Lo1g==", + "path": "microsoft.identitymodel.tokens/8.14.0", + "hashPath": "microsoft.identitymodel.tokens.8.14.0.nupkg.sha512" + }, + "NetEscapades.Configuration.Yaml/2.1.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-kNTX7kvRvbzBpLd3Vg9iu6t60tTyhVxsruAPgH6kl1GkAZIHLZw9cQysvjUenDU7JEnUgyxQnzfL8627ARDn+g==", + "path": "netescapades.configuration.yaml/2.1.0", + "hashPath": "netescapades.configuration.yaml.2.1.0.nupkg.sha512" + }, + "Pipelines.Sockets.Unofficial/2.2.8": { + "type": "package", + "serviceable": true, + "sha512": "sha512-zG2FApP5zxSx6OcdJQLbZDk2AVlN2BNQD6MorwIfV6gVj0RRxWPEp2LXAxqDGZqeNV1Zp0BNPcNaey/GXmTdvQ==", + "path": "pipelines.sockets.unofficial/2.2.8", + "hashPath": "pipelines.sockets.unofficial.2.2.8.nupkg.sha512" + }, + "Polly/7.2.4": { + "type": "package", + "serviceable": true, + "sha512": "sha512-bw00Ck5sh6ekduDE3mnCo1ohzuad946uslCDEENu3091+6UKnBuKLo4e+yaNcCzXxOZCXWY2gV4a35+K1d4LDA==", + "path": "polly/7.2.4", + "hashPath": "polly.7.2.4.nupkg.sha512" + }, + "Polly.Extensions.Http/3.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-drrG+hB3pYFY7w1c3BD+lSGYvH2oIclH8GRSehgfyP5kjnFnHKQuuBhuHLv+PWyFuaTDyk/vfRpnxOzd11+J8g==", + "path": "polly.extensions.http/3.0.0", + "hashPath": "polly.extensions.http.3.0.0.nupkg.sha512" + }, + "SharpCompress/0.41.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-z04dBVdTIAFTRKi38f0LkajaKA++bR+M8kYCbasXePILD2H+qs7CkLpyiippB24CSbTrWIgpBKm6BenZqkUwvw==", + "path": "sharpcompress/0.41.0", + "hashPath": "sharpcompress.0.41.0.nupkg.sha512" + }, + "StackExchange.Redis/2.8.24": { + "type": "package", + "serviceable": true, + "sha512": "sha512-GWllmsFAtLyhm4C47cOCipGxyEi1NQWTFUHXnJ8hiHOsK/bH3T5eLkWPVW+LRL6jDiB3g3izW3YEHgLuPoJSyA==", + "path": "stackexchange.redis/2.8.24", + "hashPath": "stackexchange.redis.2.8.24.nupkg.sha512" + }, + "System.IdentityModel.Tokens.Jwt/7.2.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-Z3Fmkrxkp+o51ANMO/PqASRRlEz8dH4mTWwZXMFMXZt2bUGztBiNcIDnwBCElYLYpzpmz4sIqHb6aW8QVLe6YQ==", + "path": "system.identitymodel.tokens.jwt/7.2.0", + "hashPath": "system.identitymodel.tokens.jwt.7.2.0.nupkg.sha512" + }, + "YamlDotNet/9.1.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-fuvGXU4Ec5HrsmEc+BiFTNPCRf1cGBI2kh/3RzMWgddM2M4ALhbSPoI3X3mhXZUD1qqQd9oSkFAtWjpz8z9eRg==", + "path": "yamldotnet/9.1.0", + "hashPath": "yamldotnet.9.1.0.nupkg.sha512" + }, + "ZstdSharp.Port/0.8.6": { + "type": "package", + "serviceable": true, + "sha512": "sha512-iP4jVLQoQmUjMU88g1WObiNr6YKZGvh4aOXn3yOJsHqZsflwRsxZPcIBvNXgjXO3vQKSLctXGLTpcBPLnWPS8A==", + "path": "zstdsharp.port/0.8.6", + "hashPath": "zstdsharp.port.0.8.6.nupkg.sha512" + }, + "StellaOps.Auth.Abstractions/1.0.0-preview.1": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Auth.Client/1.0.0-preview.1": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Auth.Security/1.0.0-preview.1": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Authority.Plugins.Abstractions/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Configuration/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Cryptography/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.DependencyInjection/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Plugin/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Scanner.Analyzers.Lang/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Scanner.Core/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + } + } +} \ No newline at end of file diff --git a/out/linknotmerge-bench.csv b/out/linknotmerge-bench.csv new file mode 100644 index 00000000..7e141e9a --- /dev/null +++ b/out/linknotmerge-bench.csv @@ -0,0 +1,4 @@ +scenario,iterations,observations,aliases,linksets,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_throughput_per_sec,min_throughput_per_sec,mean_mongo_throughput_per_sec,min_mongo_throughput_per_sec,max_allocated_mb +lnm_ingest_baseline,5,5000,500,6000,555.1984,823.4957,866.6236,366.2635,188.9349,9877.7916,5769.5175,15338.0851,8405.1257,62.4477 +lnm_ingest_fanout_medium,5,10000,800,14800,785.8909,841.6247,842.8815,453.5087,332.3822,12794.9550,11864.0639,22086.0320,20891.0579,145.8328 +lnm_ingest_fanout_high,5,15000,1200,17400,1299.3458,1367.0934,1369.9430,741.6265,557.7193,11571.0991,10949.3607,20232.5180,19781.6762,238.3450 diff --git a/out/linknotmerge-bench.json b/out/linknotmerge-bench.json new file mode 100644 index 00000000..9ac354d3 --- /dev/null +++ b/out/linknotmerge-bench.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": "linknotmerge-bench/1.0", + "capturedAt": "2025-10-26T21:09:17.6345283+00:00", + "scenarios": [ + { + "id": "lnm_ingest_baseline", + "label": "5k observations, 500 aliases", + "iterations": 5, + "observations": 5000, + "aliases": 500, + "linksets": 6000, + "meanTotalMs": 555.1983600000001, + "p95TotalMs": 823.49568, + "maxTotalMs": 866.6236, + "meanInsertMs": 366.2635, + "meanCorrelationMs": 188.93486000000001, + "meanThroughputPerSecond": 9877.791561756272, + "minThroughputPerSecond": 5769.517469868118, + "meanMongoThroughputPerSecond": 15338.085148262326, + "minMongoThroughputPerSecond": 8405.1257146248, + "maxAllocatedMb": 62.44767761230469, + "thresholdMs": 900, + "minThroughputThresholdPerSecond": 5500, + "minMongoThroughputThresholdPerSecond": 8000, + "maxAllocatedThresholdMb": 160, + "regression": { + "limit": 1.15, + "breached": false + } + }, + { + "id": "lnm_ingest_fanout_medium", + "label": "10k observations, 800 aliases", + "iterations": 5, + "observations": 10000, + "aliases": 800, + "linksets": 14800, + "meanTotalMs": 785.89092, + "p95TotalMs": 841.6247, + "maxTotalMs": 842.8815, + "meanInsertMs": 453.50868, + "meanCorrelationMs": 332.38224, + "meanThroughputPerSecond": 12794.954951406156, + "minThroughputPerSecond": 11864.063928322072, + "meanMongoThroughputPerSecond": 22086.032034175576, + "minMongoThroughputPerSecond": 20891.057937797712, + "maxAllocatedMb": 145.83282470703125, + "thresholdMs": 1300, + "minThroughputThresholdPerSecond": 8000, + "minMongoThroughputThresholdPerSecond": 13000, + "maxAllocatedThresholdMb": 220, + "regression": { + "limit": 1.15, + "breached": false + } + }, + { + "id": "lnm_ingest_fanout_high", + "label": "15k observations, 1200 aliases", + "iterations": 5, + "observations": 15000, + "aliases": 1200, + "linksets": 17400, + "meanTotalMs": 1299.3458, + "p95TotalMs": 1367.09344, + "maxTotalMs": 1369.943, + "meanInsertMs": 741.62654, + "meanCorrelationMs": 557.71926, + "meanThroughputPerSecond": 11571.099129140825, + "minThroughputPerSecond": 10949.360666830664, + "meanMongoThroughputPerSecond": 20232.5179777937, + "minMongoThroughputPerSecond": 19781.676233305086, + "maxAllocatedMb": 238.34496307373047, + "thresholdMs": 2200, + "minThroughputThresholdPerSecond": 7000, + "minMongoThroughputThresholdPerSecond": 13000, + "maxAllocatedThresholdMb": 300, + "regression": { + "limit": 1.15, + "breached": false + } + } + ] +} \ No newline at end of file diff --git a/out/linknotmerge-bench.prom b/out/linknotmerge-bench.prom new file mode 100644 index 00000000..2401ba2f --- /dev/null +++ b/out/linknotmerge-bench.prom @@ -0,0 +1,60 @@ +# HELP linknotmerge_bench_total_ms Link-Not-Merge benchmark total duration metrics (milliseconds). +# TYPE linknotmerge_bench_total_ms gauge +# HELP linknotmerge_bench_correlation_ms Link-Not-Merge benchmark correlation duration metrics (milliseconds). +# TYPE linknotmerge_bench_correlation_ms gauge +# HELP linknotmerge_bench_insert_ms Link-Not-Merge benchmark Mongo insert duration metrics (milliseconds). +# TYPE linknotmerge_bench_insert_ms gauge +# HELP linknotmerge_bench_throughput_per_sec Link-Not-Merge benchmark throughput metrics (observations per second). +# TYPE linknotmerge_bench_throughput_per_sec gauge +# HELP linknotmerge_bench_mongo_throughput_per_sec Link-Not-Merge benchmark Mongo throughput metrics (operations per second). +# TYPE linknotmerge_bench_mongo_throughput_per_sec gauge +# HELP linknotmerge_bench_allocated_mb Link-Not-Merge benchmark allocation metrics (megabytes). +# TYPE linknotmerge_bench_allocated_mb gauge +linknotmerge_bench_mean_total_ms{scenario="lnm_ingest_baseline"} 555.19836000000009 +linknotmerge_bench_p95_total_ms{scenario="lnm_ingest_baseline"} 823.49567999999999 +linknotmerge_bench_max_total_ms{scenario="lnm_ingest_baseline"} 866.62360000000001 +linknotmerge_bench_threshold_ms{scenario="lnm_ingest_baseline"} 900 +linknotmerge_bench_mean_correlation_ms{scenario="lnm_ingest_baseline"} 188.93486000000001 +linknotmerge_bench_mean_insert_ms{scenario="lnm_ingest_baseline"} 366.26350000000002 +linknotmerge_bench_mean_throughput_per_sec{scenario="lnm_ingest_baseline"} 9877.7915617562721 +linknotmerge_bench_min_throughput_per_sec{scenario="lnm_ingest_baseline"} 5769.5174698681176 +linknotmerge_bench_throughput_floor_per_sec{scenario="lnm_ingest_baseline"} 5500 +linknotmerge_bench_mean_mongo_throughput_per_sec{scenario="lnm_ingest_baseline"} 15338.085148262326 +linknotmerge_bench_min_mongo_throughput_per_sec{scenario="lnm_ingest_baseline"} 8405.1257146248008 +linknotmerge_bench_mongo_throughput_floor_per_sec{scenario="lnm_ingest_baseline"} 8000 +linknotmerge_bench_max_allocated_mb{scenario="lnm_ingest_baseline"} 62.447677612304688 +linknotmerge_bench_max_allocated_threshold_mb{scenario="lnm_ingest_baseline"} 160 +linknotmerge_bench_regression_limit{scenario="lnm_ingest_baseline"} 1.1499999999999999 +linknotmerge_bench_regression_breached{scenario="lnm_ingest_baseline"} 0 +linknotmerge_bench_mean_total_ms{scenario="lnm_ingest_fanout_medium"} 785.89092000000005 +linknotmerge_bench_p95_total_ms{scenario="lnm_ingest_fanout_medium"} 841.62469999999996 +linknotmerge_bench_max_total_ms{scenario="lnm_ingest_fanout_medium"} 842.88149999999996 +linknotmerge_bench_threshold_ms{scenario="lnm_ingest_fanout_medium"} 1300 +linknotmerge_bench_mean_correlation_ms{scenario="lnm_ingest_fanout_medium"} 332.38224000000002 +linknotmerge_bench_mean_insert_ms{scenario="lnm_ingest_fanout_medium"} 453.50868000000003 +linknotmerge_bench_mean_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 12794.954951406156 +linknotmerge_bench_min_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 11864.063928322072 +linknotmerge_bench_throughput_floor_per_sec{scenario="lnm_ingest_fanout_medium"} 8000 +linknotmerge_bench_mean_mongo_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 22086.032034175576 +linknotmerge_bench_min_mongo_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 20891.057937797712 +linknotmerge_bench_mongo_throughput_floor_per_sec{scenario="lnm_ingest_fanout_medium"} 13000 +linknotmerge_bench_max_allocated_mb{scenario="lnm_ingest_fanout_medium"} 145.83282470703125 +linknotmerge_bench_max_allocated_threshold_mb{scenario="lnm_ingest_fanout_medium"} 220 +linknotmerge_bench_regression_limit{scenario="lnm_ingest_fanout_medium"} 1.1499999999999999 +linknotmerge_bench_regression_breached{scenario="lnm_ingest_fanout_medium"} 0 +linknotmerge_bench_mean_total_ms{scenario="lnm_ingest_fanout_high"} 1299.3458000000001 +linknotmerge_bench_p95_total_ms{scenario="lnm_ingest_fanout_high"} 1367.0934400000001 +linknotmerge_bench_max_total_ms{scenario="lnm_ingest_fanout_high"} 1369.943 +linknotmerge_bench_threshold_ms{scenario="lnm_ingest_fanout_high"} 2200 +linknotmerge_bench_mean_correlation_ms{scenario="lnm_ingest_fanout_high"} 557.71925999999996 +linknotmerge_bench_mean_insert_ms{scenario="lnm_ingest_fanout_high"} 741.62653999999998 +linknotmerge_bench_mean_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 11571.099129140825 +linknotmerge_bench_min_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 10949.360666830664 +linknotmerge_bench_throughput_floor_per_sec{scenario="lnm_ingest_fanout_high"} 7000 +linknotmerge_bench_mean_mongo_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 20232.517977793701 +linknotmerge_bench_min_mongo_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 19781.676233305086 +linknotmerge_bench_mongo_throughput_floor_per_sec{scenario="lnm_ingest_fanout_high"} 13000 +linknotmerge_bench_max_allocated_mb{scenario="lnm_ingest_fanout_high"} 238.34496307373047 +linknotmerge_bench_max_allocated_threshold_mb{scenario="lnm_ingest_fanout_high"} 300 +linknotmerge_bench_regression_limit{scenario="lnm_ingest_fanout_high"} 1.1499999999999999 +linknotmerge_bench_regression_breached{scenario="lnm_ingest_fanout_high"} 0 diff --git a/out/linknotmerge-vex-bench.csv b/out/linknotmerge-vex-bench.csv new file mode 100644 index 00000000..34578590 --- /dev/null +++ b/out/linknotmerge-vex-bench.csv @@ -0,0 +1,4 @@ +scenario,iterations,observations,statements,events,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_observation_throughput_per_sec,min_observation_throughput_per_sec,mean_event_throughput_per_sec,min_event_throughput_per_sec,max_allocated_mb +vex_ingest_baseline,5,4000,24000,21326,842.8191,1319.3038,1432.7675,346.7277,496.0915,5349.8940,2791.7998,48942.4901,24653.0556,138.6365 +vex_ingest_medium,5,8000,64000,56720,1525.9929,1706.8900,1748.9056,533.3378,992.6552,5274.5883,4574.2892,57654.9190,48531.7353,326.8638 +vex_ingest_high,5,12000,120000,106910,2988.5094,3422.1728,3438.9364,903.3927,2085.1167,4066.2300,3489.4510,52456.9493,42358.0556,583.9903 diff --git a/out/linknotmerge-vex-bench.json b/out/linknotmerge-vex-bench.json new file mode 100644 index 00000000..cb140eef --- /dev/null +++ b/out/linknotmerge-vex-bench.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": "linknotmerge-vex-bench/1.0", + "capturedAt": "2025-10-26T21:29:34.4007212+00:00", + "scenarios": [ + { + "id": "vex_ingest_baseline", + "label": "4k observations, 400 aliases", + "iterations": 5, + "observations": 4000, + "statements": 24000, + "events": 21326, + "meanTotalMs": 842.81914, + "p95TotalMs": 1319.3037799999997, + "maxTotalMs": 1432.7675, + "meanInsertMs": 346.72766, + "meanCorrelationMs": 496.09147999999993, + "meanObservationThroughputPerSecond": 5349.894040882909, + "minObservationThroughputPerSecond": 2791.7997860783416, + "meanEventThroughputPerSecond": 48942.49008943273, + "minEventThroughputPerSecond": 24653.055581276763, + "maxAllocatedMb": 138.63648986816406, + "thresholdMs": 2300, + "minObservationThroughputThresholdPerSecond": 1800, + "minEventThroughputThresholdPerSecond": 2000, + "maxAllocatedThresholdMb": 220, + "regression": { + "limit": 1.15, + "breached": false + } + }, + { + "id": "vex_ingest_medium", + "label": "8k observations, 700 aliases", + "iterations": 5, + "observations": 8000, + "statements": 64000, + "events": 56720, + "meanTotalMs": 1525.99294, + "p95TotalMs": 1706.89, + "maxTotalMs": 1748.9056, + "meanInsertMs": 533.3377800000001, + "meanCorrelationMs": 992.6551599999999, + "meanObservationThroughputPerSecond": 5274.588273225903, + "minObservationThroughputPerSecond": 4574.289201201025, + "meanEventThroughputPerSecond": 57654.91903920916, + "minEventThroughputPerSecond": 48531.73532270095, + "maxAllocatedMb": 326.8638000488281, + "thresholdMs": 3200, + "minObservationThroughputThresholdPerSecond": 2200, + "minEventThroughputThresholdPerSecond": 2500, + "maxAllocatedThresholdMb": 400, + "regression": { + "limit": 1.15, + "breached": false + } + }, + { + "id": "vex_ingest_high", + "label": "12k observations, 1100 aliases", + "iterations": 5, + "observations": 12000, + "statements": 120000, + "events": 106910, + "meanTotalMs": 2988.50936, + "p95TotalMs": 3422.1728, + "maxTotalMs": 3438.9364, + "meanInsertMs": 903.3926800000002, + "meanCorrelationMs": 2085.11668, + "meanObservationThroughputPerSecond": 4066.2299506870645, + "minObservationThroughputPerSecond": 3489.450982577055, + "meanEventThroughputPerSecond": 52456.94928323016, + "minEventThroughputPerSecond": 42358.05564361166, + "maxAllocatedMb": 583.9903411865234, + "thresholdMs": 4200, + "minObservationThroughputThresholdPerSecond": 2200, + "minEventThroughputThresholdPerSecond": 2500, + "maxAllocatedThresholdMb": 700, + "regression": { + "limit": 1.15, + "breached": false + } + } + ] +} \ No newline at end of file diff --git a/out/linknotmerge-vex-bench.prom b/out/linknotmerge-vex-bench.prom new file mode 100644 index 00000000..e18eefe3 --- /dev/null +++ b/out/linknotmerge-vex-bench.prom @@ -0,0 +1,50 @@ +# HELP linknotmerge_vex_bench_total_ms Link-Not-Merge VEX benchmark total duration (milliseconds). +# TYPE linknotmerge_vex_bench_total_ms gauge +# HELP linknotmerge_vex_bench_throughput_per_sec Link-Not-Merge VEX benchmark observation throughput (observations per second). +# TYPE linknotmerge_vex_bench_throughput_per_sec gauge +# HELP linknotmerge_vex_bench_event_throughput_per_sec Link-Not-Merge VEX benchmark event throughput (events per second). +# TYPE linknotmerge_vex_bench_event_throughput_per_sec gauge +# HELP linknotmerge_vex_bench_allocated_mb Link-Not-Merge VEX benchmark max allocations (megabytes). +# TYPE linknotmerge_vex_bench_allocated_mb gauge +linknotmerge_vex_bench_mean_total_ms{scenario="vex_ingest_baseline"} 842.81913999999995 +linknotmerge_vex_bench_p95_total_ms{scenario="vex_ingest_baseline"} 1319.3037799999997 +linknotmerge_vex_bench_max_total_ms{scenario="vex_ingest_baseline"} 1432.7674999999999 +linknotmerge_vex_bench_threshold_ms{scenario="vex_ingest_baseline"} 2300 +linknotmerge_vex_bench_mean_observation_throughput_per_sec{scenario="vex_ingest_baseline"} 5349.8940408829094 +linknotmerge_vex_bench_min_observation_throughput_per_sec{scenario="vex_ingest_baseline"} 2791.7997860783416 +linknotmerge_vex_bench_observation_throughput_floor_per_sec{scenario="vex_ingest_baseline"} 1800 +linknotmerge_vex_bench_mean_event_throughput_per_sec{scenario="vex_ingest_baseline"} 48942.490089432729 +linknotmerge_vex_bench_min_event_throughput_per_sec{scenario="vex_ingest_baseline"} 24653.055581276763 +linknotmerge_vex_bench_event_throughput_floor_per_sec{scenario="vex_ingest_baseline"} 2000 +linknotmerge_vex_bench_max_allocated_mb{scenario="vex_ingest_baseline"} 138.63648986816406 +linknotmerge_vex_bench_max_allocated_threshold_mb{scenario="vex_ingest_baseline"} 220 +linknotmerge_vex_bench_regression_limit{scenario="vex_ingest_baseline"} 1.1499999999999999 +linknotmerge_vex_bench_regression_breached{scenario="vex_ingest_baseline"} 0 +linknotmerge_vex_bench_mean_total_ms{scenario="vex_ingest_medium"} 1525.9929400000001 +linknotmerge_vex_bench_p95_total_ms{scenario="vex_ingest_medium"} 1706.8900000000001 +linknotmerge_vex_bench_max_total_ms{scenario="vex_ingest_medium"} 1748.9056 +linknotmerge_vex_bench_threshold_ms{scenario="vex_ingest_medium"} 3200 +linknotmerge_vex_bench_mean_observation_throughput_per_sec{scenario="vex_ingest_medium"} 5274.5882732259033 +linknotmerge_vex_bench_min_observation_throughput_per_sec{scenario="vex_ingest_medium"} 4574.2892012010252 +linknotmerge_vex_bench_observation_throughput_floor_per_sec{scenario="vex_ingest_medium"} 2200 +linknotmerge_vex_bench_mean_event_throughput_per_sec{scenario="vex_ingest_medium"} 57654.919039209162 +linknotmerge_vex_bench_min_event_throughput_per_sec{scenario="vex_ingest_medium"} 48531.735322700952 +linknotmerge_vex_bench_event_throughput_floor_per_sec{scenario="vex_ingest_medium"} 2500 +linknotmerge_vex_bench_max_allocated_mb{scenario="vex_ingest_medium"} 326.86380004882812 +linknotmerge_vex_bench_max_allocated_threshold_mb{scenario="vex_ingest_medium"} 400 +linknotmerge_vex_bench_regression_limit{scenario="vex_ingest_medium"} 1.1499999999999999 +linknotmerge_vex_bench_regression_breached{scenario="vex_ingest_medium"} 0 +linknotmerge_vex_bench_mean_total_ms{scenario="vex_ingest_high"} 2988.50936 +linknotmerge_vex_bench_p95_total_ms{scenario="vex_ingest_high"} 3422.1727999999998 +linknotmerge_vex_bench_max_total_ms{scenario="vex_ingest_high"} 3438.9364 +linknotmerge_vex_bench_threshold_ms{scenario="vex_ingest_high"} 4200 +linknotmerge_vex_bench_mean_observation_throughput_per_sec{scenario="vex_ingest_high"} 4066.2299506870645 +linknotmerge_vex_bench_min_observation_throughput_per_sec{scenario="vex_ingest_high"} 3489.4509825770551 +linknotmerge_vex_bench_observation_throughput_floor_per_sec{scenario="vex_ingest_high"} 2200 +linknotmerge_vex_bench_mean_event_throughput_per_sec{scenario="vex_ingest_high"} 52456.949283230162 +linknotmerge_vex_bench_min_event_throughput_per_sec{scenario="vex_ingest_high"} 42358.05564361166 +linknotmerge_vex_bench_event_throughput_floor_per_sec{scenario="vex_ingest_high"} 2500 +linknotmerge_vex_bench_max_allocated_mb{scenario="vex_ingest_high"} 583.99034118652344 +linknotmerge_vex_bench_max_allocated_threshold_mb{scenario="vex_ingest_high"} 700 +linknotmerge_vex_bench_regression_limit{scenario="vex_ingest_high"} 1.1499999999999999 +linknotmerge_vex_bench_regression_breached{scenario="vex_ingest_high"} 0 diff --git a/out/notify-bench.csv b/out/notify-bench.csv new file mode 100644 index 00000000..d79c2c45 --- /dev/null +++ b/out/notify-bench.csv @@ -0,0 +1,4 @@ +scenario,iterations,events,deliveries,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb +notify_dispatch_density_05,5,5000,20000,3.4150,4.1722,4.3039,6053938.5172,4646948.1168,0.0000 +notify_dispatch_density_20,5,7500,675000,24.2274,25.8517,26.0526,27923335.5855,25909122.3141,0.0000 +notify_dispatch_density_40,5,10000,4000080,138.7387,147.7174,149.1124,28916602.9214,26825938.0172,0.0000 diff --git a/out/notify-bench.json b/out/notify-bench.json new file mode 100644 index 00000000..11c7f7a8 --- /dev/null +++ b/out/notify-bench.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": "notify-dispatch-bench/1.0", + "capturedAt": "2025-10-26T20:28:56.3603045+00:00", + "scenarios": [ + { + "id": "notify_dispatch_density_05", + "label": "50 rules / 5% fanout", + "iterations": 5, + "totalEvents": 5000, + "totalRules": 50, + "actionsPerRule": 2, + "averageMatchesPerEvent": 2, + "minMatchesPerEvent": 2, + "maxMatchesPerEvent": 2, + "averageDeliveriesPerEvent": 4, + "totalDeliveries": 20000, + "meanMs": 3.41498, + "p95Ms": 4.17216, + "maxMs": 4.3039, + "meanThroughputPerSecond": 6053938.51717893, + "minThroughputPerSecond": 4646948.116824276, + "maxAllocatedMb": 0, + "thresholdMs": 400, + "minThroughputThresholdPerSecond": 15000, + "maxAllocatedThresholdMb": 128, + "regression": { + "limit": 1.15, + "breached": false + } + }, + { + "id": "notify_dispatch_density_20", + "label": "150 rules / 20% fanout", + "iterations": 5, + "totalEvents": 7500, + "totalRules": 150, + "actionsPerRule": 3, + "averageMatchesPerEvent": 30, + "minMatchesPerEvent": 30, + "maxMatchesPerEvent": 30, + "averageDeliveriesPerEvent": 90, + "totalDeliveries": 675000, + "meanMs": 24.2274, + "p95Ms": 25.85172, + "maxMs": 26.0526, + "meanThroughputPerSecond": 27923335.585545264, + "minThroughputPerSecond": 25909122.314087655, + "maxAllocatedMb": 0, + "thresholdMs": 650, + "minThroughputThresholdPerSecond": 30000, + "maxAllocatedThresholdMb": 192, + "regression": { + "limit": 1.15, + "breached": false + } + }, + { + "id": "notify_dispatch_density_40", + "label": "300 rules / 40% fanout", + "iterations": 5, + "totalEvents": 10000, + "totalRules": 300, + "actionsPerRule": 4, + "averageMatchesPerEvent": 100.002, + "minMatchesPerEvent": 60, + "maxMatchesPerEvent": 120, + "averageDeliveriesPerEvent": 400.008, + "totalDeliveries": 4000080, + "meanMs": 138.73866, + "p95Ms": 147.71738000000002, + "maxMs": 149.1124, + "meanThroughputPerSecond": 28916602.921385907, + "minThroughputPerSecond": 26825938.017227273, + "maxAllocatedMb": 0, + "thresholdMs": 900, + "minThroughputThresholdPerSecond": 45000, + "maxAllocatedThresholdMb": 256, + "regression": { + "limit": 1.15, + "breached": false + } + } + ] +} \ No newline at end of file diff --git a/out/notify-bench.prom b/out/notify-bench.prom new file mode 100644 index 00000000..27f97f19 --- /dev/null +++ b/out/notify-bench.prom @@ -0,0 +1,39 @@ +# HELP notify_dispatch_bench_duration_ms Notify dispatch benchmark duration metrics (milliseconds). +# TYPE notify_dispatch_bench_duration_ms gauge +# HELP notify_dispatch_bench_throughput_per_sec Notify dispatch benchmark throughput metrics (deliveries per second). +# TYPE notify_dispatch_bench_throughput_per_sec gauge +# HELP notify_dispatch_bench_allocation_mb Notify dispatch benchmark allocation metrics (megabytes). +# TYPE notify_dispatch_bench_allocation_mb gauge +notify_dispatch_bench_mean_ms{scenario="notify_dispatch_density_05"} 3.4149799999999999 +notify_dispatch_bench_p95_ms{scenario="notify_dispatch_density_05"} 4.1721599999999999 +notify_dispatch_bench_max_ms{scenario="notify_dispatch_density_05"} 4.3038999999999996 +notify_dispatch_bench_threshold_ms{scenario="notify_dispatch_density_05"} 400 +notify_dispatch_bench_mean_throughput_per_sec{scenario="notify_dispatch_density_05"} 6053938.5171789303 +notify_dispatch_bench_min_throughput_per_sec{scenario="notify_dispatch_density_05"} 4646948.1168242758 +notify_dispatch_bench_min_throughput_threshold_per_sec{scenario="notify_dispatch_density_05"} 15000 +notify_dispatch_bench_max_allocated_mb{scenario="notify_dispatch_density_05"} 0 +notify_dispatch_bench_max_allocated_threshold_mb{scenario="notify_dispatch_density_05"} 128 +notify_dispatch_bench_regression_limit{scenario="notify_dispatch_density_05"} 1.1499999999999999 +notify_dispatch_bench_regression_breached{scenario="notify_dispatch_density_05"} 0 +notify_dispatch_bench_mean_ms{scenario="notify_dispatch_density_20"} 24.227399999999999 +notify_dispatch_bench_p95_ms{scenario="notify_dispatch_density_20"} 25.85172 +notify_dispatch_bench_max_ms{scenario="notify_dispatch_density_20"} 26.052600000000002 +notify_dispatch_bench_threshold_ms{scenario="notify_dispatch_density_20"} 650 +notify_dispatch_bench_mean_throughput_per_sec{scenario="notify_dispatch_density_20"} 27923335.585545264 +notify_dispatch_bench_min_throughput_per_sec{scenario="notify_dispatch_density_20"} 25909122.314087655 +notify_dispatch_bench_min_throughput_threshold_per_sec{scenario="notify_dispatch_density_20"} 30000 +notify_dispatch_bench_max_allocated_mb{scenario="notify_dispatch_density_20"} 0 +notify_dispatch_bench_max_allocated_threshold_mb{scenario="notify_dispatch_density_20"} 192 +notify_dispatch_bench_regression_limit{scenario="notify_dispatch_density_20"} 1.1499999999999999 +notify_dispatch_bench_regression_breached{scenario="notify_dispatch_density_20"} 0 +notify_dispatch_bench_mean_ms{scenario="notify_dispatch_density_40"} 138.73866000000001 +notify_dispatch_bench_p95_ms{scenario="notify_dispatch_density_40"} 147.71738000000002 +notify_dispatch_bench_max_ms{scenario="notify_dispatch_density_40"} 149.11240000000001 +notify_dispatch_bench_threshold_ms{scenario="notify_dispatch_density_40"} 900 +notify_dispatch_bench_mean_throughput_per_sec{scenario="notify_dispatch_density_40"} 28916602.921385907 +notify_dispatch_bench_min_throughput_per_sec{scenario="notify_dispatch_density_40"} 26825938.017227273 +notify_dispatch_bench_min_throughput_threshold_per_sec{scenario="notify_dispatch_density_40"} 45000 +notify_dispatch_bench_max_allocated_mb{scenario="notify_dispatch_density_40"} 0 +notify_dispatch_bench_max_allocated_threshold_mb{scenario="notify_dispatch_density_40"} 256 +notify_dispatch_bench_regression_limit{scenario="notify_dispatch_density_40"} 1.1499999999999999 +notify_dispatch_bench_regression_breached{scenario="notify_dispatch_density_40"} 0 diff --git a/out/policy-bench.csv b/out/policy-bench.csv new file mode 100644 index 00000000..79cdb0d4 --- /dev/null +++ b/out/policy-bench.csv @@ -0,0 +1,2 @@ +scenario,iterations,findings,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb +policy_eval_baseline,3,1000000,1109.3542,1257.7493,1280.1721,912094.5581,781144.9726,563.6901 diff --git a/out/policy-bench.json b/out/policy-bench.json new file mode 100644 index 00000000..ddf1fc54 --- /dev/null +++ b/out/policy-bench.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": "policy-bench/1.0", + "capturedAt": "2025-10-26T19:57:27.4363234+00:00", + "scenarios": [ + { + "id": "policy_eval_baseline", + "label": "Policy evaluation (100k components, 1M findings)", + "iterations": 3, + "findingCount": 1000000, + "meanMs": 1109.3542333333335, + "p95Ms": 1257.74929, + "maxMs": 1280.1721, + "meanThroughputPerSecond": 912094.5580512757, + "minThroughputPerSecond": 781144.9726173537, + "maxAllocatedMb": 563.6900634765625, + "thresholdMs": 20000, + "minThroughputThresholdPerSecond": 60000, + "maxAllocatedThresholdMb": 900, + "regression": { + "limit": 1.15, + "breached": false + } + } + ] +} \ No newline at end of file diff --git a/out/policy-bench.prom b/out/policy-bench.prom new file mode 100644 index 00000000..b9ecd0c0 --- /dev/null +++ b/out/policy-bench.prom @@ -0,0 +1,17 @@ +# HELP policy_engine_bench_duration_ms Policy Engine benchmark duration metrics (milliseconds). +# TYPE policy_engine_bench_duration_ms gauge +# HELP policy_engine_bench_throughput_per_sec Policy Engine benchmark throughput metrics (findings per second). +# TYPE policy_engine_bench_throughput_per_sec gauge +# HELP policy_engine_bench_allocation_mb Policy Engine benchmark allocation metrics (megabytes). +# TYPE policy_engine_bench_allocation_mb gauge +policy_engine_bench_mean_ms{scenario="policy_eval_baseline"} 1109.3542333333335 +policy_engine_bench_p95_ms{scenario="policy_eval_baseline"} 1257.74929 +policy_engine_bench_max_ms{scenario="policy_eval_baseline"} 1280.1721 +policy_engine_bench_threshold_ms{scenario="policy_eval_baseline"} 20000 +policy_engine_bench_mean_throughput_per_sec{scenario="policy_eval_baseline"} 912094.55805127567 +policy_engine_bench_min_throughput_per_sec{scenario="policy_eval_baseline"} 781144.97261735366 +policy_engine_bench_min_throughput_threshold_per_sec{scenario="policy_eval_baseline"} 60000 +policy_engine_bench_max_allocated_mb{scenario="policy_eval_baseline"} 563.6900634765625 +policy_engine_bench_max_allocated_threshold_mb{scenario="policy_eval_baseline"} 900 +policy_engine_bench_regression_limit{scenario="policy_eval_baseline"} 1.1499999999999999 +policy_engine_bench_regression_breached{scenario="policy_eval_baseline"} 0 diff --git a/out/policy-simulations/policy-simulation-summary.json b/out/policy-simulations/policy-simulation-summary.json new file mode 100644 index 00000000..57e092e5 --- /dev/null +++ b/out/policy-simulations/policy-simulation-summary.json @@ -0,0 +1,32 @@ +[ + { + "ScenarioName": "baseline", + "Success": true, + "ChangedCount": 2, + "Failures": [], + "ActualStatuses": { + "library:pkg/openssl@1.1.1w": "Blocked", + "library:pkg/internal-runtime@1.0.0": "Warned" + } + }, + { + "ScenarioName": "internal-only", + "Success": true, + "ChangedCount": 2, + "Failures": [], + "ActualStatuses": { + "library:pkg/internal-app@2.0.0": "RequiresVex", + "library:pkg/kev-component@3.1.4": "RequiresVex" + } + }, + { + "ScenarioName": "serverless", + "Success": true, + "ChangedCount": 2, + "Failures": [], + "ActualStatuses": { + "library:pkg/aws-lambda@1.0.0": "Blocked", + "image:sha256:untrusted-base": "Blocked" + } + } +] \ No newline at end of file diff --git a/samples/TASKS.md b/samples/TASKS.md index 01c5d5cb..92fb292b 100644 --- a/samples/TASKS.md +++ b/samples/TASKS.md @@ -9,15 +9,13 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SAMPLES-POLICY-20-001 | TODO | Samples Guild, Policy Guild | POLICY-ENGINE-20-002, DOCS-POLICY-20-011 | Create sample policies (`baseline.pol`, `serverless.pol`, `internal-only.pol`) with annotated SBOM/advisory fixtures. | Samples stored under `samples/policy/`; README documents usage; tests validate deterministic outputs. | -| SAMPLES-POLICY-20-002 | TODO | Samples Guild, UI Guild | UI-POLICY-20-002 | Produce simulation diff fixtures (before/after JSON) for UI/CLI tests. | Fixtures committed with schema validation; referenced by UI+CLI tests; docs cross-link. | +| SAMPLES-POLICY-20-001 | DONE (2025-10-26) | Samples Guild, Policy Guild | POLICY-ENGINE-20-002, DOCS-POLICY-20-011 | Create sample policies (`baseline.pol`, `serverless.pol`, `internal-only.pol`) with annotated SBOM/advisory fixtures. | Samples stored under `samples/policy/`; README documents usage; tests validate deterministic outputs. | +| SAMPLES-POLICY-20-002 | DONE (2025-10-26) | Samples Guild, UI Guild | UI-POLICY-20-002 | Produce simulation diff fixtures (before/after JSON) for UI/CLI tests. | Fixtures committed with schema validation; referenced by UI+CLI tests; docs cross-link. | ## Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SAMPLES-GRAPH-21-001 | TODO | Samples Guild, Cartographer Guild | CARTO-GRAPH-21-003 | Produce small/medium SBOM graph fixtures (JSON, GraphML, layout tiles) for automated tests and docs. | Fixtures stored under `samples/graph/`; validated by Cartographer + UI tests; README documents usage. | -| SAMPLES-GRAPH-21-002 | TODO | Samples Guild, UI Guild | UI-GRAPH-21-005 | Capture golden Graph Explorer screenshots (baseline/diff) and path exports for visual regression + documentation. | Screenshots exported; stored with metadata; referenced in docs; tests consume assets. | ## Link-Not-Merge v1 diff --git a/samples/api/scheduler/graph-build-job.json b/samples/api/scheduler/graph-build-job.json new file mode 100644 index 00000000..c75184f3 --- /dev/null +++ b/samples/api/scheduler/graph-build-job.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "scheduler.graph-build-job@1", + "id": "gbj_20251026a", + "tenantId": "tenant-alpha", + "sbomId": "sbom_20251026", + "sbomVersionId": "sbom_ver_20251026", + "sbomDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "graphSnapshotId": "graph_snap_20251026", + "status": "running", + "trigger": "sbom-version", + "attempts": 1, + "cartographerJobId": "carto_job_42", + "correlationId": "evt_svc_987", + "createdAt": "2025-10-26T12:00:00+00:00", + "startedAt": "2025-10-26T12:00:05+00:00", + "metadata": { + "sbomEventId": "sbom_evt_20251026" + } +} diff --git a/samples/api/scheduler/graph-overlay-job.json b/samples/api/scheduler/graph-overlay-job.json new file mode 100644 index 00000000..f6242514 --- /dev/null +++ b/samples/api/scheduler/graph-overlay-job.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": "scheduler.graph-overlay-job@1", + "id": "goj_20251026a", + "tenantId": "tenant-alpha", + "graphSnapshotId": "graph_snap_20251026", + "buildJobId": "gbj_20251026a", + "overlayKind": "policy", + "overlayKey": "policy@2025-10-01", + "subjects": [ + "artifact:service-api", + "artifact:service-worker" + ], + "status": "queued", + "trigger": "policy", + "attempts": 0, + "correlationId": "policy_run_321", + "createdAt": "2025-10-26T12:05:00+00:00", + "metadata": { + "policyRunId": "policy_run_321" + } +} diff --git a/samples/api/scheduler/policy-diff-summary.json b/samples/api/scheduler/policy-diff-summary.json new file mode 100644 index 00000000..a8560c1b --- /dev/null +++ b/samples/api/scheduler/policy-diff-summary.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": "scheduler.policy-diff-summary@1", + "added": 12, + "removed": 8, + "unchanged": 657, + "bySeverity": { + "critical": { + "up": 1 + }, + "high": { + "up": 3, + "down": 4 + }, + "medium": { + "up": 2, + "down": 1 + } + }, + "ruleHits": [ + { + "ruleId": "rule-block-critical", + "ruleName": "Block Critical Findings", + "up": 1 + }, + { + "ruleId": "rule-quiet-low", + "ruleName": "Quiet Low Risk", + "down": 2 + } + ] +} diff --git a/samples/api/scheduler/policy-explain-trace.json b/samples/api/scheduler/policy-explain-trace.json new file mode 100644 index 00000000..834ecd76 --- /dev/null +++ b/samples/api/scheduler/policy-explain-trace.json @@ -0,0 +1,83 @@ +{ + "schemaVersion": "scheduler.policy-explain-trace@1", + "findingId": "finding:sbom:S-42/pkg:npm/lodash@4.17.21", + "policyId": "P-7", + "policyVersion": 4, + "tenantId": "default", + "runId": "run:P-7:2025-10-26:auto", + "evaluatedAt": "2025-10-26T14:06:01+00:00", + "verdict": { + "status": "blocked", + "severity": "critical", + "score": 19.5, + "rationale": "Matches rule-block-critical" + }, + "ruleChain": [ + { + "ruleId": "rule-allow-known", + "ruleName": "Allow Known Vendors", + "action": "allow", + "decision": "skipped", + "condition": "when vendor == \"trusted\"" + }, + { + "ruleId": "rule-block-critical", + "ruleName": "Block Critical Findings", + "action": "block", + "decision": "matched", + "score": 19.5, + "condition": "when severity >= Critical" + } + ], + "evidence": [ + { + "type": "advisory", + "reference": "CVE-2025-12345", + "source": "nvd", + "status": "affected", + "weight": 1, + "justification": "Vendor advisory", + "metadata": {} + }, + { + "type": "vex", + "reference": "vex:ghsa-2025-0001", + "source": "vendor", + "status": "not_affected", + "weight": 0.5, + "justification": "Runtime unreachable", + "metadata": { + "justificationid": "csaf:justification/123" + } + } + ], + "vexImpacts": [ + { + "statementId": "vex:ghsa-2025-0001", + "provider": "vendor", + "status": "not_affected", + "accepted": true, + "justification": "Runtime unreachable", + "confidence": "medium" + } + ], + "history": [ + { + "status": "blocked", + "occurredAt": "2025-10-26T14:06:01+00:00", + "actor": "policy-engine", + "note": "Initial evaluation" + }, + { + "status": "blocked", + "occurredAt": "2025-10-26T14:16:01+00:00", + "actor": "policy-engine", + "note": "Replay verification" + } + ], + "metadata": { + "componentpurl": "pkg:npm/lodash@4.17.21", + "sbomid": "sbom:S-42", + "traceid": "01HE0BJX5S4T9YCN6ZT0" + } +} diff --git a/samples/api/scheduler/policy-run-request.json b/samples/api/scheduler/policy-run-request.json new file mode 100644 index 00000000..d46c7a63 --- /dev/null +++ b/samples/api/scheduler/policy-run-request.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": "scheduler.policy-run-request@1", + "tenantId": "default", + "policyId": "P-7", + "policyVersion": 4, + "mode": "incremental", + "priority": "normal", + "runId": "run:P-7:2025-10-26:auto", + "queuedAt": "2025-10-26T14:05:00+00:00", + "requestedBy": "user:cli", + "correlationId": "req-20251026T140500Z", + "metadata": { + "source": "stella policy run", + "trigger": "cli" + }, + "inputs": { + "sbomSet": [ + "sbom:S-318", + "sbom:S-42" + ], + "advisoryCursor": "2025-10-26T13:59:00+00:00", + "vexCursor": "2025-10-26T13:58:30+00:00", + "environment": { + "exposure": "internet", + "sealed": false + }, + "captureExplain": true + } +} diff --git a/samples/api/scheduler/policy-run-status.json b/samples/api/scheduler/policy-run-status.json new file mode 100644 index 00000000..c5a46631 --- /dev/null +++ b/samples/api/scheduler/policy-run-status.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": "scheduler.policy-run-status@1", + "runId": "run:P-7:2025-10-26:auto", + "tenantId": "default", + "policyId": "P-7", + "policyVersion": 4, + "mode": "incremental", + "status": "succeeded", + "priority": "normal", + "queuedAt": "2025-10-26T14:05:00+00:00", + "startedAt": "2025-10-26T14:05:11+00:00", + "finishedAt": "2025-10-26T14:06:01+00:00", + "determinismHash": "sha256:e3c2b2f3b1aa4567890abcdef1234567890abcdef1234567890abcdef123456", + "traceId": "01HE0BJX5S4T9YCN6ZT0", + "explainUri": "blob://policy/P-7/runs/2025-10-26T14-06-01Z.json", + "metadata": { + "orchestrator": "scheduler", + "sbombatchhash": "sha256:abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234" + }, + "stats": { + "components": 1742, + "rulesFired": 68023, + "findingsWritten": 4321, + "vexOverrides": 210, + "quieted": 12, + "durationSeconds": 50.8 + }, + "inputs": { + "sbomSet": [ + "sbom:S-318", + "sbom:S-42" + ], + "advisoryCursor": "2025-10-26T13:59:00+00:00", + "vexCursor": "2025-10-26T13:58:30+00:00", + "environment": { + "exposure": "internet", + "sealed": false + }, + "captureExplain": true + } +} diff --git a/samples/api/scheduler/run-summary.json b/samples/api/scheduler/run-summary.json new file mode 100644 index 00000000..c9734ebd --- /dev/null +++ b/samples/api/scheduler/run-summary.json @@ -0,0 +1,101 @@ +{ + "tenantId": "tenant-alpha", + "scheduleId": "sch_20251018a", + "updatedAt": "2025-10-18T22:10:10Z", + "lastRun": { + "runId": "run_20251018_0001", + "trigger": "feedser", + "state": "completed", + "createdAt": "2025-10-18T22:03:14Z", + "startedAt": "2025-10-18T22:03:20Z", + "finishedAt": "2025-10-18T22:08:45Z", + "stats": { + "candidates": 1280, + "deduped": 910, + "queued": 0, + "completed": 910, + "deltas": 42, + "newCriticals": 7, + "newHigh": 11, + "newMedium": 18, + "newLow": 6 + }, + "error": null + }, + "recent": [ + { + "runId": "run_20251018_0001", + "trigger": "feedser", + "state": "completed", + "createdAt": "2025-10-18T22:03:14Z", + "startedAt": "2025-10-18T22:03:20Z", + "finishedAt": "2025-10-18T22:08:45Z", + "stats": { + "candidates": 1280, + "deduped": 910, + "queued": 0, + "completed": 910, + "deltas": 42, + "newCriticals": 7, + "newHigh": 11, + "newMedium": 18, + "newLow": 6 + }, + "error": null + }, + { + "runId": "run_20251017_0003", + "trigger": "cron", + "state": "error", + "createdAt": "2025-10-17T22:01:02Z", + "startedAt": "2025-10-17T22:01:08Z", + "finishedAt": "2025-10-17T22:04:11Z", + "stats": { + "candidates": 1040, + "deduped": 812, + "queued": 0, + "completed": 640, + "deltas": 18, + "newCriticals": 2, + "newHigh": 4, + "newMedium": 7, + "newLow": 3 + }, + "error": "scanner timeout" + }, + { + "runId": "run_20251016_0007", + "trigger": "manual", + "state": "cancelled", + "createdAt": "2025-10-16T20:00:00Z", + "startedAt": "2025-10-16T20:00:04Z", + "finishedAt": null, + "stats": { + "candidates": 820, + "deduped": 640, + "queued": 0, + "completed": 0, + "deltas": 0, + "newCriticals": 0, + "newHigh": 0, + "newMedium": 0, + "newLow": 0 + }, + "error": null + } + ], + "counters": { + "total": 3, + "planning": 0, + "queued": 0, + "running": 0, + "completed": 1, + "error": 1, + "cancelled": 1, + "totalDeltas": 60, + "totalNewCriticals": 9, + "totalNewHigh": 15, + "totalNewMedium": 25, + "totalNewLow": 9 + } +} diff --git a/samples/policy/README.md b/samples/policy/README.md new file mode 100644 index 00000000..fdb6d6b8 --- /dev/null +++ b/samples/policy/README.md @@ -0,0 +1,25 @@ +# Policy Samples + +Curated fixtures used by CI smoke/determinism checks and example documentation. + +| Scenario | Policy | Findings | Expected Diff | UI/CLI Diff Fixture | +|----------|--------|----------|---------------|---------------------| +| `baseline` | `docs/examples/policies/baseline.yaml` | `samples/policy/baseline/findings.json` | `samples/policy/baseline/diffs.json` | `samples/policy/simulations/baseline/diff.json` | +| `serverless` | `docs/examples/policies/serverless.yaml` | `samples/policy/serverless/findings.json` | `samples/policy/serverless/diffs.json` | `samples/policy/simulations/serverless/diff.json` | +| `internal-only` | `docs/examples/policies/internal-only.yaml` | `samples/policy/internal-only/findings.json` | `samples/policy/internal-only/diffs.json` | `samples/policy/simulations/internal-only/diff.json` | + +Run the simulation harness locally: + +```bash +dotnet run \ + --project tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj \ + -- \ + --scenario-root samples/policy/simulations \ + --output out/policy-simulations +``` + +Then inspect `out/policy-simulations/policy-simulation-summary.json` for verdict changes. + +--- + +*Last updated: 2025-10-26.* diff --git a/samples/policy/baseline/diffs.json b/samples/policy/baseline/diffs.json new file mode 100644 index 00000000..ddf10cb0 --- /dev/null +++ b/samples/policy/baseline/diffs.json @@ -0,0 +1,12 @@ +[ + { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Blocked", + "rule": "block_critical" + }, + { + "findingId": "library:pkg/internal-runtime@1.0.0", + "status": "Warned", + "rule": "alert_warn_eol_runtime" + } +] diff --git a/samples/policy/baseline/findings.json b/samples/policy/baseline/findings.json new file mode 100644 index 00000000..565e7688 --- /dev/null +++ b/samples/policy/baseline/findings.json @@ -0,0 +1,14 @@ +[ + { + "findingId": "library:pkg/openssl@1.1.1w", + "severity": "Critical", + "source": "NVD", + "environment": "internet" + }, + { + "findingId": "library:pkg/internal-runtime@1.0.0", + "severity": "Low", + "source": "NVD", + "tags": ["runtime:eol"] + } +] diff --git a/samples/policy/internal-only/diffs.json b/samples/policy/internal-only/diffs.json new file mode 100644 index 00000000..749e20c3 --- /dev/null +++ b/samples/policy/internal-only/diffs.json @@ -0,0 +1,12 @@ +[ + { + "findingId": "library:pkg/internal-app@2.0.0", + "status": "RequiresVex", + "rule": "accept_vendor_vex" + }, + { + "findingId": "library:pkg/kev-component@3.1.4", + "status": "RequiresVex", + "rule": "accept_vendor_vex" + } +] diff --git a/samples/policy/internal-only/findings.json b/samples/policy/internal-only/findings.json new file mode 100644 index 00000000..c8cdb5a8 --- /dev/null +++ b/samples/policy/internal-only/findings.json @@ -0,0 +1,15 @@ +[ + { + "findingId": "library:pkg/internal-app@2.0.0", + "severity": "Medium", + "source": "GHSA", + "environment": "internal" + }, + { + "findingId": "library:pkg/kev-component@3.1.4", + "severity": "High", + "source": "NVD", + "tags": ["kev"], + "environment": "internal" + } +] diff --git a/samples/policy/serverless/diffs.json b/samples/policy/serverless/diffs.json new file mode 100644 index 00000000..d956d08f --- /dev/null +++ b/samples/policy/serverless/diffs.json @@ -0,0 +1,12 @@ +[ + { + "findingId": "library:pkg/aws-lambda@1.0.0", + "status": "Blocked", + "rule": "block_any_high" + }, + { + "findingId": "image:sha256:untrusted-base", + "status": "Blocked", + "rule": "forbid_unpinned_base" + } +] diff --git a/samples/policy/serverless/findings.json b/samples/policy/serverless/findings.json new file mode 100644 index 00000000..23a3fdc4 --- /dev/null +++ b/samples/policy/serverless/findings.json @@ -0,0 +1,15 @@ +[ + { + "findingId": "library:pkg/aws-lambda@1.0.0", + "severity": "High", + "source": "NVD", + "environment": "serverless" + }, + { + "findingId": "image:sha256:untrusted-base", + "severity": "Medium", + "source": "NVD", + "tags": ["image:latest-tag"], + "environment": "serverless" + } +] diff --git a/samples/policy/simulations/baseline/diff.json b/samples/policy/simulations/baseline/diff.json new file mode 100644 index 00000000..3bc68419 --- /dev/null +++ b/samples/policy/simulations/baseline/diff.json @@ -0,0 +1,23 @@ +{ + "summary": { + "policy": "baseline", + "policyDigest": "sha256:simulation-baseline", + "changed": 2 + }, + "diffs": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "baselineStatus": "Pass", + "projectedStatus": "Blocked", + "rule": "block_critical", + "notes": "Critical severity must be remediated before deploy." + }, + { + "findingId": "library:pkg/internal-runtime@1.0.0", + "baselineStatus": "Pass", + "projectedStatus": "Warned", + "rule": "alert_warn_eol_runtime", + "notes": "Runtime marked as EOL; upgrade recommended." + } + ] +} diff --git a/samples/policy/simulations/baseline/scenario.json b/samples/policy/simulations/baseline/scenario.json new file mode 100644 index 00000000..b96342af --- /dev/null +++ b/samples/policy/simulations/baseline/scenario.json @@ -0,0 +1,21 @@ +{ + "name": "baseline", + "policyPath": "docs/examples/policies/baseline.yaml", + "findings": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "severity": "Critical", + "source": "NVD" + }, + { + "findingId": "library:pkg/internal-runtime@1.0.0", + "severity": "Low", + "source": "NVD", + "tags": ["runtime:eol"] + } + ], + "expectedDiffs": [ + { "findingId": "library:pkg/openssl@1.1.1w", "status": "Blocked" }, + { "findingId": "library:pkg/internal-runtime@1.0.0", "status": "Warned" } + ] +} diff --git a/samples/policy/simulations/internal-only/diff.json b/samples/policy/simulations/internal-only/diff.json new file mode 100644 index 00000000..4d730801 --- /dev/null +++ b/samples/policy/simulations/internal-only/diff.json @@ -0,0 +1,23 @@ +{ + "summary": { + "policy": "internal-only", + "policyDigest": "sha256:simulation-internal-only", + "changed": 2 + }, + "diffs": [ + { + "findingId": "library:pkg/internal-app@2.0.0", + "baselineStatus": "Pass", + "projectedStatus": "RequiresVex", + "rule": "accept_vendor_vex", + "notes": "Trust vendor VEX statements for internal scope." + }, + { + "findingId": "library:pkg/kev-component@3.1.4", + "baselineStatus": "Pass", + "projectedStatus": "RequiresVex", + "rule": "accept_vendor_vex", + "notes": "Trust vendor VEX statements for internal scope." + } + ] +} diff --git a/samples/policy/simulations/internal-only/scenario.json b/samples/policy/simulations/internal-only/scenario.json new file mode 100644 index 00000000..12711308 --- /dev/null +++ b/samples/policy/simulations/internal-only/scenario.json @@ -0,0 +1,23 @@ +{ + "name": "internal-only", + "policyPath": "docs/examples/policies/internal-only.yaml", + "findings": [ + { + "findingId": "library:pkg/internal-app@2.0.0", + "severity": "Medium", + "source": "GHSA", + "environment": "internal" + }, + { + "findingId": "library:pkg/kev-component@3.1.4", + "severity": "High", + "source": "NVD", + "tags": ["kev"], + "environment": "internal" + } + ], + "expectedDiffs": [ + { "findingId": "library:pkg/internal-app@2.0.0", "status": "RequiresVex" }, + { "findingId": "library:pkg/kev-component@3.1.4", "status": "RequiresVex" } + ] +} diff --git a/samples/policy/simulations/serverless/diff.json b/samples/policy/simulations/serverless/diff.json new file mode 100644 index 00000000..3ff0de3c --- /dev/null +++ b/samples/policy/simulations/serverless/diff.json @@ -0,0 +1,23 @@ +{ + "summary": { + "policy": "serverless", + "policyDigest": "sha256:simulation-serverless", + "changed": 2 + }, + "diffs": [ + { + "findingId": "library:pkg/aws-lambda@1.0.0", + "baselineStatus": "Pass", + "projectedStatus": "Blocked", + "rule": "block_any_high", + "notes": "Serverless workloads block High+ severities." + }, + { + "findingId": "image:sha256:untrusted-base", + "baselineStatus": "Pass", + "projectedStatus": "Blocked", + "rule": "forbid_unpinned_base", + "notes": "Base image must be pinned (no :latest)." + } + ] +} diff --git a/samples/policy/simulations/serverless/scenario.json b/samples/policy/simulations/serverless/scenario.json new file mode 100644 index 00000000..8c465196 --- /dev/null +++ b/samples/policy/simulations/serverless/scenario.json @@ -0,0 +1,23 @@ +{ + "name": "serverless", + "policyPath": "docs/examples/policies/serverless.yaml", + "findings": [ + { + "findingId": "library:pkg/aws-lambda@1.0.0", + "severity": "High", + "source": "NVD", + "environment": "serverless" + }, + { + "findingId": "image:sha256:untrusted-base", + "severity": "Medium", + "source": "NVD", + "tags": ["image:latest-tag"], + "environment": "serverless" + } + ], + "expectedDiffs": [ + { "findingId": "library:pkg/aws-lambda@1.0.0", "status": "Blocked" }, + { "findingId": "image:sha256:untrusted-base", "status": "Blocked" } + ] +} diff --git a/scripts/export-policy-schemas.sh b/scripts/export-policy-schemas.sh new file mode 100644 index 00000000..58dc46eb --- /dev/null +++ b/scripts/export-policy-schemas.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" +OUTPUT_DIR="${1:-$REPO_ROOT/docs/schemas}" + +pushd "$REPO_ROOT" > /dev/null + +dotnet run --project tools/PolicySchemaExporter -- "$OUTPUT_DIR" + +popd > /dev/null diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 7ca4da55..47a1dfb7 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,4 +1,4 @@ - + $(SolutionDir)StellaOps.Concelier.PluginBinaries $(MSBuildThisFileDirectory)StellaOps.Concelier.PluginBinaries @@ -35,15 +35,15 @@ - + - - - - - - - - - + + + + + + + + + diff --git a/src/StellaOps.Aoc.Tests/AocWriteGuardTests.cs b/src/StellaOps.Aoc.Tests/AocWriteGuardTests.cs new file mode 100644 index 00000000..cf4632de --- /dev/null +++ b/src/StellaOps.Aoc.Tests/AocWriteGuardTests.cs @@ -0,0 +1,113 @@ +using System.Text.Json; +using StellaOps.Aoc; + +namespace StellaOps.Aoc.Tests; + +public sealed class AocWriteGuardTests +{ + private static readonly AocWriteGuard Guard = new(); + + [Fact] + public void Validate_ReturnsSuccess_ForMinimalValidDocument() + { + using var document = JsonDocument.Parse(""" + { + "tenant": "default", + "source": {"vendor": "osv"}, + "upstream": { + "upstream_id": "GHSA-xxxx", + "content_hash": "sha256:abc", + "signature": { "present": false } + }, + "content": { + "format": "OSV", + "raw": {"id": "GHSA-xxxx"} + }, + "linkset": {} + } + """); + + var result = Guard.Validate(document.RootElement); + + Assert.True(result.IsValid); + Assert.Empty(result.Violations); + } + + [Fact] + public void Validate_FlagsMissingTenant() + { + using var document = JsonDocument.Parse(""" + { + "source": {"vendor": "osv"}, + "upstream": { + "upstream_id": "GHSA-xxxx", + "content_hash": "sha256:abc", + "signature": { "present": false } + }, + "content": { + "format": "OSV", + "raw": {"id": "GHSA-xxxx"} + }, + "linkset": {} + } + """); + + var result = Guard.Validate(document.RootElement); + + Assert.False(result.IsValid); + Assert.Contains(result.Violations, v => v.ErrorCode == "ERR_AOC_004" && v.Path == "/tenant"); + } + + [Fact] + public void Validate_FlagsForbiddenField() + { + using var document = JsonDocument.Parse(""" + { + "tenant": "default", + "severity": "high", + "source": {"vendor": "osv"}, + "upstream": { + "upstream_id": "GHSA-xxxx", + "content_hash": "sha256:abc", + "signature": { "present": false } + }, + "content": { + "format": "OSV", + "raw": {"id": "GHSA-xxxx"} + }, + "linkset": {} + } + """); + + var result = Guard.Validate(document.RootElement); + + Assert.False(result.IsValid); + Assert.Contains(result.Violations, v => v.ErrorCode == "ERR_AOC_001" && v.Path == "/severity"); + } + + [Fact] + public void Validate_FlagsInvalidSignatureMetadata() + { + using var document = JsonDocument.Parse(""" + { + "tenant": "default", + "source": {"vendor": "osv"}, + "upstream": { + "upstream_id": "GHSA-xxxx", + "content_hash": "sha256:abc", + "signature": { "present": true, "format": "dsse" } + }, + "content": { + "format": "OSV", + "raw": {"id": "GHSA-xxxx"} + }, + "linkset": {} + } + """); + + var result = Guard.Validate(document.RootElement); + + Assert.False(result.IsValid); + Assert.Contains(result.Violations, v => v.ErrorCode == "ERR_AOC_005" && v.Path.Contains("/sig")); + } +} diff --git a/src/StellaOps.Aoc.Tests/StellaOps.Aoc.Tests.csproj b/src/StellaOps.Aoc.Tests/StellaOps.Aoc.Tests.csproj new file mode 100644 index 00000000..1c8328a3 --- /dev/null +++ b/src/StellaOps.Aoc.Tests/StellaOps.Aoc.Tests.csproj @@ -0,0 +1,41 @@ + + + + net10.0 + preview + enable + enable + true + Exe + false + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Aoc.Tests/UnitTest1.cs b/src/StellaOps.Aoc.Tests/UnitTest1.cs new file mode 100644 index 00000000..515425ec --- /dev/null +++ b/src/StellaOps.Aoc.Tests/UnitTest1.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Aoc.Tests; + +public class UnitTest1 +{ + [Fact] + public void Test1() + { + + } +} diff --git a/src/StellaOps.Aoc.Tests/xunit.runner.json b/src/StellaOps.Aoc.Tests/xunit.runner.json new file mode 100644 index 00000000..86c7ea05 --- /dev/null +++ b/src/StellaOps.Aoc.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/src/StellaOps.Aoc/AocForbiddenKeys.cs b/src/StellaOps.Aoc/AocForbiddenKeys.cs new file mode 100644 index 00000000..1ad51f67 --- /dev/null +++ b/src/StellaOps.Aoc/AocForbiddenKeys.cs @@ -0,0 +1,25 @@ +using System.Collections.Immutable; + +namespace StellaOps.Aoc; + +public static class AocForbiddenKeys +{ + private static readonly ImmutableHashSet ForbiddenTopLevel = new[] + { + "severity", + "cvss", + "cvss_vector", + "effective_status", + "effective_range", + "merged_from", + "consensus_provider", + "reachability", + "asset_criticality", + "risk_score", + }.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase); + + public static bool IsForbiddenTopLevel(string propertyName) => ForbiddenTopLevel.Contains(propertyName); + + public static bool IsDerivedField(string propertyName) + => propertyName.StartsWith("effective_", StringComparison.OrdinalIgnoreCase); +} diff --git a/src/StellaOps.Aoc/AocGuardException.cs b/src/StellaOps.Aoc/AocGuardException.cs new file mode 100644 index 00000000..3c0db884 --- /dev/null +++ b/src/StellaOps.Aoc/AocGuardException.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Aoc; + +public sealed class AocGuardException : Exception +{ + public AocGuardException(AocGuardResult result) + : base("AOC guard validation failed.") + { + Result = result ?? throw new ArgumentNullException(nameof(result)); + } + + public AocGuardResult Result { get; } + + public ImmutableArray Violations => Result.Violations; +} diff --git a/src/StellaOps.Aoc/AocGuardExtensions.cs b/src/StellaOps.Aoc/AocGuardExtensions.cs new file mode 100644 index 00000000..f6083c39 --- /dev/null +++ b/src/StellaOps.Aoc/AocGuardExtensions.cs @@ -0,0 +1,22 @@ +using System.Text.Json; + +namespace StellaOps.Aoc; + +public static class AocGuardExtensions +{ + public static AocGuardResult ValidateOrThrow(this IAocGuard guard, JsonElement document, AocGuardOptions? options = null) + { + if (guard is null) + { + throw new ArgumentNullException(nameof(guard)); + } + + var result = guard.Validate(document, options); + if (!result.IsValid) + { + throw new AocGuardException(result); + } + + return result; + } +} diff --git a/src/StellaOps.Aoc/AocGuardOptions.cs b/src/StellaOps.Aoc/AocGuardOptions.cs new file mode 100644 index 00000000..27e46af1 --- /dev/null +++ b/src/StellaOps.Aoc/AocGuardOptions.cs @@ -0,0 +1,29 @@ +using System.Collections.Immutable; + +namespace StellaOps.Aoc; + +public sealed record AocGuardOptions +{ + private static readonly ImmutableHashSet DefaultRequiredTopLevel = new[] + { + "tenant", + "source", + "upstream", + "content", + "linkset", + }.ToImmutableHashSet(StringComparer.OrdinalIgnoreCase); + + public static AocGuardOptions Default { get; } = new(); + + public ImmutableHashSet RequiredTopLevelFields { get; init; } = DefaultRequiredTopLevel; + + /// + /// When true, signature metadata is required under upstream.signature. + /// + public bool RequireSignatureMetadata { get; init; } = true; + + /// + /// When true, tenant must be a non-empty string. + /// + public bool RequireTenant { get; init; } = true; +} diff --git a/src/StellaOps.Aoc/AocGuardResult.cs b/src/StellaOps.Aoc/AocGuardResult.cs new file mode 100644 index 00000000..9ea1a3b1 --- /dev/null +++ b/src/StellaOps.Aoc/AocGuardResult.cs @@ -0,0 +1,14 @@ +using System.Collections.Immutable; + +namespace StellaOps.Aoc; + +public sealed record AocGuardResult(bool IsValid, ImmutableArray Violations) +{ + public static AocGuardResult Success { get; } = new(true, ImmutableArray.Empty); + + public static AocGuardResult FromViolations(IEnumerable violations) + { + var array = violations.ToImmutableArray(); + return array.IsDefaultOrEmpty ? Success : new(false, array); + } +} diff --git a/src/StellaOps.Aoc/AocViolation.cs b/src/StellaOps.Aoc/AocViolation.cs new file mode 100644 index 00000000..e9337566 --- /dev/null +++ b/src/StellaOps.Aoc/AocViolation.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Aoc; + +public sealed record AocViolation( + [property: JsonPropertyName("code")] AocViolationCode Code, + [property: JsonPropertyName("errorCode")] string ErrorCode, + [property: JsonPropertyName("path")] string Path, + [property: JsonPropertyName("message")] string Message) +{ + public static AocViolation Create(AocViolationCode code, string path, string message) + => new(code, code.ToErrorCode(), path, message); +} diff --git a/src/StellaOps.Aoc/AocViolationCode.cs b/src/StellaOps.Aoc/AocViolationCode.cs new file mode 100644 index 00000000..7b31555e --- /dev/null +++ b/src/StellaOps.Aoc/AocViolationCode.cs @@ -0,0 +1,34 @@ +namespace StellaOps.Aoc; + +public enum AocViolationCode +{ + None = 0, + ForbiddenField, + MergeAttempt, + IdempotencyViolation, + MissingProvenance, + SignatureInvalid, + DerivedFindingDetected, + UnknownField, + MissingRequiredField, + InvalidTenant, + InvalidSignatureMetadata, +} + +public static class AocViolationCodeExtensions +{ + public static string ToErrorCode(this AocViolationCode code) => code switch + { + AocViolationCode.ForbiddenField => "ERR_AOC_001", + AocViolationCode.MergeAttempt => "ERR_AOC_002", + AocViolationCode.IdempotencyViolation => "ERR_AOC_003", + AocViolationCode.MissingProvenance => "ERR_AOC_004", + AocViolationCode.SignatureInvalid => "ERR_AOC_005", + AocViolationCode.DerivedFindingDetected => "ERR_AOC_006", + AocViolationCode.UnknownField => "ERR_AOC_007", + AocViolationCode.MissingRequiredField => "ERR_AOC_004", + AocViolationCode.InvalidTenant => "ERR_AOC_004", + AocViolationCode.InvalidSignatureMetadata => "ERR_AOC_005", + _ => "ERR_AOC_000", + }; +} diff --git a/src/StellaOps.Aoc/AocWriteGuard.cs b/src/StellaOps.Aoc/AocWriteGuard.cs new file mode 100644 index 00000000..861b34d5 --- /dev/null +++ b/src/StellaOps.Aoc/AocWriteGuard.cs @@ -0,0 +1,127 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json; + +namespace StellaOps.Aoc; + +public interface IAocGuard +{ + AocGuardResult Validate(JsonElement document, AocGuardOptions? options = null); +} + +public sealed class AocWriteGuard : IAocGuard +{ + public AocGuardResult Validate(JsonElement document, AocGuardOptions? options = null) + { + options ??= AocGuardOptions.Default; + var violations = ImmutableArray.CreateBuilder(); + var presentTopLevel = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var property in document.EnumerateObject()) + { + presentTopLevel.Add(property.Name); + + if (AocForbiddenKeys.IsForbiddenTopLevel(property.Name)) + { + violations.Add(AocViolation.Create(AocViolationCode.ForbiddenField, $"/{property.Name}", $"Field '{property.Name}' is forbidden in AOC documents.")); + continue; + } + + if (AocForbiddenKeys.IsDerivedField(property.Name)) + { + violations.Add(AocViolation.Create(AocViolationCode.DerivedFindingDetected, $"/{property.Name}", $"Derived field '{property.Name}' must not be written during ingestion.")); + } + } + + foreach (var required in options.RequiredTopLevelFields) + { + if (!document.TryGetProperty(required, out var element) || element.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined) + { + violations.Add(AocViolation.Create(AocViolationCode.MissingRequiredField, $"/{required}", $"Required field '{required}' is missing.")); + continue; + } + + if (options.RequireTenant && string.Equals(required, "tenant", StringComparison.OrdinalIgnoreCase)) + { + if (element.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(element.GetString())) + { + violations.Add(AocViolation.Create(AocViolationCode.InvalidTenant, "/tenant", "Tenant must be a non-empty string.")); + } + } + } + + if (document.TryGetProperty("upstream", out var upstream) && upstream.ValueKind == JsonValueKind.Object) + { + if (!upstream.TryGetProperty("content_hash", out var contentHash) || contentHash.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(contentHash.GetString())) + { + violations.Add(AocViolation.Create(AocViolationCode.MissingProvenance, "/upstream/content_hash", "Upstream content hash is required.")); + } + + if (!upstream.TryGetProperty("signature", out var signature) || signature.ValueKind != JsonValueKind.Object) + { + if (options.RequireSignatureMetadata) + { + violations.Add(AocViolation.Create(AocViolationCode.MissingProvenance, "/upstream/signature", "Signature metadata is required.")); + } + } + else if (options.RequireSignatureMetadata) + { + ValidateSignature(signature, violations); + } + } + else + { + violations.Add(AocViolation.Create(AocViolationCode.MissingRequiredField, "/upstream", "Upstream metadata is required.")); + } + + if (document.TryGetProperty("content", out var content) && content.ValueKind == JsonValueKind.Object) + { + if (!content.TryGetProperty("raw", out var raw) || raw.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined) + { + violations.Add(AocViolation.Create(AocViolationCode.MissingProvenance, "/content/raw", "Raw upstream payload must be preserved.")); + } + } + else + { + violations.Add(AocViolation.Create(AocViolationCode.MissingRequiredField, "/content", "Content metadata is required.")); + } + + if (!document.TryGetProperty("linkset", out var linkset) || linkset.ValueKind != JsonValueKind.Object) + { + violations.Add(AocViolation.Create(AocViolationCode.MissingRequiredField, "/linkset", "Linkset metadata is required.")); + } + + return AocGuardResult.FromViolations(violations); + } + + private static void ValidateSignature(JsonElement signature, ImmutableArray.Builder violations) + { + if (!signature.TryGetProperty("present", out var presentElement) || presentElement.ValueKind is not (JsonValueKind.True or JsonValueKind.False)) + { + violations.Add(AocViolation.Create(AocViolationCode.InvalidSignatureMetadata, "/upstream/signature/present", "Signature metadata must include 'present' boolean.")); + return; + } + + var signaturePresent = presentElement.GetBoolean(); + + if (!signaturePresent) + { + return; + } + + if (!signature.TryGetProperty("format", out var formatElement) || formatElement.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(formatElement.GetString())) + { + violations.Add(AocViolation.Create(AocViolationCode.InvalidSignatureMetadata, "/upstream/signature/format", "Signature format is required when signature is present.")); + } + + if (!signature.TryGetProperty("sig", out var sigElement) || sigElement.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(sigElement.GetString())) + { + violations.Add(AocViolation.Create(AocViolationCode.SignatureInvalid, "/upstream/signature/sig", "Signature payload is required when signature is present.")); + } + + if (!signature.TryGetProperty("key_id", out var keyIdElement) || keyIdElement.ValueKind != JsonValueKind.String || string.IsNullOrWhiteSpace(keyIdElement.GetString())) + { + violations.Add(AocViolation.Create(AocViolationCode.InvalidSignatureMetadata, "/upstream/signature/key_id", "Signature key identifier is required when signature is present.")); + } + } +} diff --git a/src/StellaOps.Aoc/ServiceCollectionExtensions.cs b/src/StellaOps.Aoc/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..ac9369d3 --- /dev/null +++ b/src/StellaOps.Aoc/ServiceCollectionExtensions.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.Aoc; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddAocGuard(this IServiceCollection services) + { + if (services is null) + { + throw new ArgumentNullException(nameof(services)); + } + + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Aoc/StellaOps.Aoc.csproj b/src/StellaOps.Aoc/StellaOps.Aoc.csproj new file mode 100644 index 00000000..febd1315 --- /dev/null +++ b/src/StellaOps.Aoc/StellaOps.Aoc.csproj @@ -0,0 +1,12 @@ + + + net10.0 + preview + enable + enable + true + + + + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj index 054613bd..ff30e46c 100644 --- a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj @@ -1,21 +1,21 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj index 5f7f5ec6..dcc734a2 100644 --- a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj @@ -1,25 +1,25 @@ - - - net10.0 - preview - enable - enable - true - false - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj index 62c77cc6..a0f87bb7 100644 --- a/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj +++ b/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj @@ -1,30 +1,30 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj b/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj index ce4ae69f..c585f6c7 100644 --- a/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj +++ b/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj @@ -1,38 +1,38 @@ - - - net10.0 - preview - enable - enable - true - - - Sender-constrained authentication primitives (DPoP, mTLS) shared across StellaOps services. - StellaOps.Auth.Security - StellaOps - StellaOps - stellaops;dpop;mtls;oauth2;security - AGPL-3.0-or-later - https://stella-ops.org - https://git.stella-ops.org/stella-ops.org/git.stella-ops.org - git - true - true - true - snupkg - README.md - 1.0.0-preview.1 - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + Sender-constrained authentication primitives (DPoP, mTLS) shared across StellaOps services. + StellaOps.Auth.Security + StellaOps + StellaOps + stellaops;dpop;mtls;oauth2;security + AGPL-3.0-or-later + https://stella-ops.org + https://git.stella-ops.org/stella-ops.org/git.stella-ops.org + git + true + true + true + snupkg + README.md + 1.0.0-preview.1 + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsScopesTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsScopesTests.cs new file mode 100644 index 00000000..50405723 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsScopesTests.cs @@ -0,0 +1,38 @@ +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Auth.Abstractions.Tests; + +public class StellaOpsScopesTests +{ + [Theory] + [InlineData(StellaOpsScopes.AdvisoryRead)] + [InlineData(StellaOpsScopes.AdvisoryIngest)] + [InlineData(StellaOpsScopes.VexRead)] + [InlineData(StellaOpsScopes.VexIngest)] + [InlineData(StellaOpsScopes.AocVerify)] + [InlineData(StellaOpsScopes.PolicyWrite)] + [InlineData(StellaOpsScopes.PolicySubmit)] + [InlineData(StellaOpsScopes.PolicyApprove)] + [InlineData(StellaOpsScopes.PolicyRun)] + [InlineData(StellaOpsScopes.FindingsRead)] + [InlineData(StellaOpsScopes.EffectiveWrite)] + [InlineData(StellaOpsScopes.GraphRead)] + [InlineData(StellaOpsScopes.VulnRead)] + [InlineData(StellaOpsScopes.GraphWrite)] + [InlineData(StellaOpsScopes.GraphExport)] + [InlineData(StellaOpsScopes.GraphSimulate)] + public void All_IncludesNewScopes(string scope) + { + Assert.Contains(scope, StellaOpsScopes.All); + } + + [Theory] + [InlineData("Advisory:Read", StellaOpsScopes.AdvisoryRead)] + [InlineData(" VEX:Ingest ", StellaOpsScopes.VexIngest)] + [InlineData("AOC:VERIFY", StellaOpsScopes.AocVerify)] + public void Normalize_NormalizesToLowerCase(string input, string expected) + { + Assert.Equal(expected, StellaOpsScopes.Normalize(input)); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs index 7028669b..be704fc2 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs @@ -24,29 +24,125 @@ public static class StellaOpsScopes public const string AuthorityUsersManage = "authority.users.manage"; /// - /// Scope granting administrative access to Authority client registrations. - /// - public const string AuthorityClientsManage = "authority.clients.manage"; - - /// - /// Scope granting read-only access to Authority audit logs. - /// - public const string AuthorityAuditRead = "authority.audit.read"; - - /// - /// Synthetic scope representing trusted network bypass. - /// - public const string Bypass = "stellaops.bypass"; - - private static readonly HashSet KnownScopes = new(StringComparer.OrdinalIgnoreCase) - { - ConcelierJobsTrigger, - ConcelierMerge, - AuthorityUsersManage, - AuthorityClientsManage, - AuthorityAuditRead, - Bypass - }; + /// Scope granting administrative access to Authority client registrations. + /// + public const string AuthorityClientsManage = "authority.clients.manage"; + + /// + /// Scope granting read-only access to Authority audit logs. + /// + public const string AuthorityAuditRead = "authority.audit.read"; + + /// + /// Synthetic scope representing trusted network bypass. + /// + public const string Bypass = "stellaops.bypass"; + + /// + /// Scope granting read-only access to raw advisory ingestion data. + /// + public const string AdvisoryRead = "advisory:read"; + + /// + /// Scope granting write access for raw advisory ingestion. + /// + public const string AdvisoryIngest = "advisory:ingest"; + + /// + /// Scope granting read-only access to raw VEX ingestion data. + /// + public const string VexRead = "vex:read"; + + /// + /// Scope granting write access for raw VEX ingestion. + /// + public const string VexIngest = "vex:ingest"; + + /// + /// Scope granting permission to execute aggregation-only contract verification. + /// + public const string AocVerify = "aoc:verify"; + + /// + /// Scope granting permission to create or edit policy drafts. + /// + public const string PolicyWrite = "policy:write"; + + /// + /// Scope granting permission to submit drafts for review. + /// + public const string PolicySubmit = "policy:submit"; + + /// + /// Scope granting permission to approve or reject policies. + /// + public const string PolicyApprove = "policy:approve"; + + /// + /// Scope granting permission to trigger policy runs and activation workflows. + /// + public const string PolicyRun = "policy:run"; + + /// + /// Scope granting read-only access to effective findings materialised by Policy Engine. + /// + public const string FindingsRead = "findings:read"; + + /// + /// Scope granted to Policy Engine service identity for writing effective findings. + /// + public const string EffectiveWrite = "effective:write"; + + /// + /// Scope granting read-only access to graph queries and overlays. + /// + public const string GraphRead = "graph:read"; + + /// + /// Scope granting read-only access to Vuln Explorer resources and permalinks. + /// + public const string VulnRead = "vuln:read"; + + /// + /// Scope granting permission to enqueue or mutate graph build jobs. + /// + public const string GraphWrite = "graph:write"; + + /// + /// Scope granting permission to export graph artefacts (GraphML/JSONL/etc.). + /// + public const string GraphExport = "graph:export"; + + /// + /// Scope granting permission to trigger what-if simulations on graphs. + /// + public const string GraphSimulate = "graph:simulate"; + + private static readonly HashSet KnownScopes = new(StringComparer.OrdinalIgnoreCase) + { + ConcelierJobsTrigger, + ConcelierMerge, + AuthorityUsersManage, + AuthorityClientsManage, + AuthorityAuditRead, + Bypass, + AdvisoryRead, + AdvisoryIngest, + VexRead, + VexIngest, + AocVerify, + PolicyWrite, + PolicySubmit, + PolicyApprove, + PolicyRun, + FindingsRead, + EffectiveWrite, + GraphRead, + VulnRead, + GraphWrite, + GraphExport, + GraphSimulate + }; /// /// Normalises a scope string (trim/convert to lower case). diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsServiceIdentities.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsServiceIdentities.cs new file mode 100644 index 00000000..f1923a3b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsServiceIdentities.cs @@ -0,0 +1,17 @@ +namespace StellaOps.Auth.Abstractions; + +/// +/// Canonical identifiers for StellaOps service principals. +/// +public static class StellaOpsServiceIdentities +{ + /// + /// Service identity used by Policy Engine when materialising effective findings. + /// + public const string PolicyEngine = "policy-engine"; + + /// + /// Service identity used by Cartographer when constructing and maintaining graph projections. + /// + public const string Cartographer = "cartographer"; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj index f52435fc..e2e8acdc 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOps.Auth.Client.Tests.csproj @@ -9,7 +9,7 @@ - - + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj index 90e81d6f..9b864848 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj +++ b/src/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj @@ -30,10 +30,10 @@ - + - + diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs index 69731659..09c4c8a1 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs @@ -1,50 +1,55 @@ -using System; -using System.Net; -using StellaOps.Auth.ServerIntegration; -using Xunit; - -namespace StellaOps.Auth.ServerIntegration.Tests; - -public class StellaOpsResourceServerOptionsTests -{ - [Fact] - public void Validate_NormalisesCollections() - { - var options = new StellaOpsResourceServerOptions - { - Authority = "https://authority.stella-ops.test", - BackchannelTimeout = TimeSpan.FromSeconds(10), - TokenClockSkew = TimeSpan.FromSeconds(30) - }; - - options.Audiences.Add(" api://concelier "); - options.Audiences.Add("api://concelier"); - options.Audiences.Add("api://concelier-admin"); - - options.RequiredScopes.Add(" Concelier.Jobs.Trigger "); - options.RequiredScopes.Add("concelier.jobs.trigger"); - options.RequiredScopes.Add("AUTHORITY.USERS.MANAGE"); - - options.BypassNetworks.Add("127.0.0.1/32"); - options.BypassNetworks.Add(" 127.0.0.1/32 "); - options.BypassNetworks.Add("::1/128"); - - options.Validate(); - - Assert.Equal(new Uri("https://authority.stella-ops.test"), options.AuthorityUri); - Assert.Equal(new[] { "api://concelier", "api://concelier-admin" }, options.Audiences); - Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, options.NormalizedScopes); - Assert.True(options.BypassMatcher.IsAllowed(IPAddress.Parse("127.0.0.1"))); - Assert.True(options.BypassMatcher.IsAllowed(IPAddress.IPv6Loopback)); - } - - [Fact] - public void Validate_Throws_When_AuthorityMissing() - { - var options = new StellaOpsResourceServerOptions(); - - var exception = Assert.Throws(() => options.Validate()); - - Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); - } -} +using System; +using System.Net; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class StellaOpsResourceServerOptionsTests +{ + [Fact] + public void Validate_NormalisesCollections() + { + var options = new StellaOpsResourceServerOptions + { + Authority = "https://authority.stella-ops.test", + BackchannelTimeout = TimeSpan.FromSeconds(10), + TokenClockSkew = TimeSpan.FromSeconds(30) + }; + + options.Audiences.Add(" api://concelier "); + options.Audiences.Add("api://concelier"); + options.Audiences.Add("api://concelier-admin"); + + options.RequiredScopes.Add(" Concelier.Jobs.Trigger "); + options.RequiredScopes.Add("concelier.jobs.trigger"); + options.RequiredScopes.Add("AUTHORITY.USERS.MANAGE"); + + options.RequiredTenants.Add(" Tenant-Alpha "); + options.RequiredTenants.Add("tenant-alpha"); + options.RequiredTenants.Add("Tenant-Beta"); + + options.BypassNetworks.Add("127.0.0.1/32"); + options.BypassNetworks.Add(" 127.0.0.1/32 "); + options.BypassNetworks.Add("::1/128"); + + options.Validate(); + + Assert.Equal(new Uri("https://authority.stella-ops.test"), options.AuthorityUri); + Assert.Equal(new[] { "api://concelier", "api://concelier-admin" }, options.Audiences); + Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, options.NormalizedScopes); + Assert.Equal(new[] { "tenant-alpha", "tenant-beta" }, options.NormalizedTenants); + Assert.True(options.BypassMatcher.IsAllowed(IPAddress.Parse("127.0.0.1"))); + Assert.True(options.BypassMatcher.IsAllowed(IPAddress.IPv6Loopback)); + } + + [Fact] + public void Validate_Throws_When_AuthorityMissing() + { + var options = new StellaOpsResourceServerOptions(); + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs index b2efc9ee..76339c8e 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs @@ -1,123 +1,199 @@ -using System; -using System.Net; -using System.Security.Claims; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Auth.Abstractions; -using StellaOps.Auth.ServerIntegration; -using Xunit; - -namespace StellaOps.Auth.ServerIntegration.Tests; - -public class StellaOpsScopeAuthorizationHandlerTests -{ - [Fact] - public async Task HandleRequirement_Succeeds_WhenScopePresent() - { - var optionsMonitor = CreateOptionsMonitor(options => - { - options.Authority = "https://authority.example"; - options.Validate(); - }); - - var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1")); - var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); - var principal = new StellaOpsPrincipalBuilder() - .WithSubject("user-1") - .WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger }) - .Build(); - - var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); - - await handler.HandleAsync(context); - - Assert.True(context.HasSucceeded); - } - - [Fact] - public async Task HandleRequirement_Succeeds_WhenBypassNetworkMatches() - { - var optionsMonitor = CreateOptionsMonitor(options => - { - options.Authority = "https://authority.example"; - options.BypassNetworks.Add("127.0.0.1/32"); - options.Validate(); - }); - - var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("127.0.0.1")); - var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); - var principal = new ClaimsPrincipal(new ClaimsIdentity()); - var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); - - await handler.HandleAsync(context); - - Assert.True(context.HasSucceeded); - } - - [Fact] - public async Task HandleRequirement_Fails_WhenScopeMissingAndNoBypass() - { - var optionsMonitor = CreateOptionsMonitor(options => - { - options.Authority = "https://authority.example"; - options.Validate(); - }); - - var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("203.0.113.10")); - var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); - var principal = new ClaimsPrincipal(new ClaimsIdentity()); - var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); - - await handler.HandleAsync(context); - - Assert.False(context.HasSucceeded); - } - - private static (StellaOpsScopeAuthorizationHandler Handler, IHttpContextAccessor Accessor) CreateHandler(IOptionsMonitor optionsMonitor, IPAddress remoteAddress) - { - var accessor = new HttpContextAccessor(); - var httpContext = new DefaultHttpContext(); - httpContext.Connection.RemoteIpAddress = remoteAddress; - accessor.HttpContext = httpContext; - - var bypassEvaluator = new StellaOpsBypassEvaluator(optionsMonitor, NullLogger.Instance); - - var handler = new StellaOpsScopeAuthorizationHandler( - accessor, - bypassEvaluator, - NullLogger.Instance); - return (handler, accessor); - } - - private static IOptionsMonitor CreateOptionsMonitor(Action configure) - => new TestOptionsMonitor(configure); - - private sealed class TestOptionsMonitor : IOptionsMonitor - where TOptions : class, new() - { - private readonly TOptions value; - - public TestOptionsMonitor(Action configure) - { - value = new TOptions(); - configure(value); - } - - public TOptions CurrentValue => value; - - public TOptions Get(string? name) => value; - - public IDisposable OnChange(Action listener) => NullDisposable.Instance; - - private sealed class NullDisposable : IDisposable - { - public static NullDisposable Instance { get; } = new(); - public void Dispose() - { - } - } - } -} +using System; +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class StellaOpsScopeAuthorizationHandlerTests +{ + [Fact] + public async Task HandleRequirement_Succeeds_WhenScopePresent() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.RequiredTenants.Add("tenant-alpha"); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new StellaOpsPrincipalBuilder() + .WithSubject("user-1") + .WithTenant("tenant-alpha") + .WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger }) + .Build(); + + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + [Fact] + [Fact] + public async Task HandleRequirement_Fails_WhenTenantMismatch() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.RequiredTenants.Add("tenant-alpha"); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new StellaOpsPrincipalBuilder() + .WithSubject("user-1") + .WithTenant("tenant-beta") + .WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger }) + .Build(); + + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.False(context.HasSucceeded); + } + + public async Task HandleRequirement_Succeeds_WhenBypassNetworkMatches() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.BypassNetworks.Add("127.0.0.1/32"); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("127.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new ClaimsPrincipal(new ClaimsIdentity()); + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Fails_WhenScopeMissingAndNoBypass() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("203.0.113.10")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new ClaimsPrincipal(new ClaimsIdentity()); + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.False(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Fails_WhenDefaultScopeMissing() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.RequiredScopes.Add(StellaOpsScopes.PolicyRun); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("198.51.100.5")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new StellaOpsPrincipalBuilder() + .WithSubject("user-tenant") + .WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger }) + .Build(); + + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.False(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Succeeds_WhenDefaultScopePresent() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.RequiredScopes.Add(StellaOpsScopes.PolicyRun); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("198.51.100.5")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new StellaOpsPrincipalBuilder() + .WithSubject("user-tenant") + .WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger, StellaOpsScopes.PolicyRun }) + .Build(); + + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + private static (StellaOpsScopeAuthorizationHandler Handler, IHttpContextAccessor Accessor) CreateHandler(IOptionsMonitor optionsMonitor, IPAddress remoteAddress) + { + var accessor = new HttpContextAccessor(); + var httpContext = new DefaultHttpContext(); + httpContext.Connection.RemoteIpAddress = remoteAddress; + accessor.HttpContext = httpContext; + + var bypassEvaluator = new StellaOpsBypassEvaluator(optionsMonitor, NullLogger.Instance); + + var handler = new StellaOpsScopeAuthorizationHandler( + accessor, + bypassEvaluator, + optionsMonitor, + NullLogger.Instance); + return (handler, accessor); + } + + private static IOptionsMonitor CreateOptionsMonitor(Action configure) + => new TestOptionsMonitor(configure); + + private sealed class TestOptionsMonitor : IOptionsMonitor + where TOptions : class, new() + { + private readonly TOptions value; + + public TestOptionsMonitor(Action configure) + { + value = new TOptions(); + configure(value); + } + + public TOptions CurrentValue => value; + + public TOptions Get(string? name) => value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static NullDisposable Instance { get; } = new(); + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs index f4957457..bb68fb62 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsResourceServerOptions.cs @@ -12,6 +12,7 @@ public sealed class StellaOpsResourceServerOptions { private readonly List audiences = new(); private readonly List requiredScopes = new(); + private readonly List requiredTenants = new(); private readonly List bypassNetworks = new(); /// @@ -34,6 +35,11 @@ public sealed class StellaOpsResourceServerOptions /// public IList RequiredScopes => requiredScopes; + /// + /// Tenants permitted to access the resource server (empty list disables tenant checks). + /// + public IList RequiredTenants => requiredTenants; + /// /// Networks permitted to bypass authentication (used for trusted on-host automation). /// @@ -64,6 +70,11 @@ public sealed class StellaOpsResourceServerOptions /// public IReadOnlyList NormalizedScopes { get; private set; } = Array.Empty(); + /// + /// Gets the normalised tenant list (populated during validation). + /// + public IReadOnlyList NormalizedTenants { get; private set; } = Array.Empty(); + /// /// Gets the network matcher used for bypass checks (populated during validation). /// @@ -105,12 +116,17 @@ public sealed class StellaOpsResourceServerOptions NormalizeList(audiences, toLower: false); NormalizeList(requiredScopes, toLower: true); + NormalizeList(requiredTenants, toLower: true); NormalizeList(bypassNetworks, toLower: false); NormalizedScopes = requiredScopes.Count == 0 ? Array.Empty() : requiredScopes.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + NormalizedTenants = requiredTenants.Count == 0 + ? Array.Empty() + : requiredTenants.OrderBy(static tenant => tenant, StringComparer.Ordinal).ToArray(); + BypassMatcher = bypassNetworks.Count == 0 ? NetworkMaskMatcher.DenyAll : new NetworkMaskMatcher(bypassNetworks); diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs index 519950f3..21152b22 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/StellaOpsScopeAuthorizationHandler.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Security.Claims; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using StellaOps.Auth.Abstractions; namespace StellaOps.Auth.ServerIntegration; @@ -16,15 +18,18 @@ internal sealed class StellaOpsScopeAuthorizationHandler : AuthorizationHandler< { private readonly IHttpContextAccessor httpContextAccessor; private readonly StellaOpsBypassEvaluator bypassEvaluator; + private readonly IOptionsMonitor optionsMonitor; private readonly ILogger logger; public StellaOpsScopeAuthorizationHandler( IHttpContextAccessor httpContextAccessor, StellaOpsBypassEvaluator bypassEvaluator, + IOptionsMonitor optionsMonitor, ILogger logger) { this.httpContextAccessor = httpContextAccessor; this.bypassEvaluator = bypassEvaluator; + this.optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor)); this.logger = logger; } @@ -32,25 +37,47 @@ internal sealed class StellaOpsScopeAuthorizationHandler : AuthorizationHandler< AuthorizationHandlerContext context, StellaOpsScopeRequirement requirement) { + var resourceOptions = optionsMonitor.CurrentValue; + var httpContext = httpContextAccessor.HttpContext; + var combinedScopes = CombineRequiredScopes(resourceOptions.NormalizedScopes, requirement.RequiredScopes); HashSet? userScopes = null; if (context.User?.Identity?.IsAuthenticated == true) { userScopes = ExtractScopes(context.User); - foreach (var scope in requirement.RequiredScopes) + foreach (var scope in combinedScopes) { - if (userScopes.Contains(scope)) + if (!userScopes.Contains(scope)) + { + continue; + } + + if (TenantAllowed(context.User, resourceOptions, out var normalizedTenant)) { context.Succeed(requirement); return Task.CompletedTask; } + + if (logger.IsEnabled(LogLevel.Debug)) + { + var allowedTenants = resourceOptions.NormalizedTenants.Count == 0 + ? "(none)" + : string.Join(", ", resourceOptions.NormalizedTenants); + + logger.LogDebug( + "Tenant requirement not satisfied. RequiredTenants={RequiredTenants}; PrincipalTenant={PrincipalTenant}; Remote={Remote}", + allowedTenants, + normalizedTenant ?? "(none)", + httpContext?.Connection.RemoteIpAddress); + } + + // tenant mismatch cannot be resolved by checking additional scopes for this principal + break; } } - var httpContext = httpContextAccessor.HttpContext; - - if (httpContext is not null && bypassEvaluator.ShouldBypass(httpContext, requirement.RequiredScopes)) + if (httpContext is not null && bypassEvaluator.ShouldBypass(httpContext, combinedScopes)) { context.Succeed(requirement); return Task.CompletedTask; @@ -58,21 +85,51 @@ internal sealed class StellaOpsScopeAuthorizationHandler : AuthorizationHandler< if (logger.IsEnabled(LogLevel.Debug)) { - var required = string.Join(", ", requirement.RequiredScopes); + var required = string.Join(", ", combinedScopes); var principalScopes = userScopes is null || userScopes.Count == 0 ? "(none)" : string.Join(", ", userScopes); + var tenantValue = context.User?.FindFirstValue(StellaOpsClaimTypes.Tenant) ?? "(none)"; logger.LogDebug( - "Scope requirement not satisfied. Required={RequiredScopes}; PrincipalScopes={PrincipalScopes}; Remote={Remote}", + "Scope requirement not satisfied. Required={RequiredScopes}; PrincipalScopes={PrincipalScopes}; Tenant={Tenant}; Remote={Remote}", required, principalScopes, + tenantValue, httpContext?.Connection.RemoteIpAddress); } return Task.CompletedTask; } + private static bool TenantAllowed(ClaimsPrincipal principal, StellaOpsResourceServerOptions options, out string? normalizedTenant) + { + normalizedTenant = null; + + if (options.NormalizedTenants.Count == 0) + { + return true; + } + + var rawTenant = principal.FindFirstValue(StellaOpsClaimTypes.Tenant); + if (string.IsNullOrWhiteSpace(rawTenant)) + { + return false; + } + + normalizedTenant = rawTenant.Trim().ToLowerInvariant(); + + foreach (var allowed in options.NormalizedTenants) + { + if (string.Equals(allowed, normalizedTenant, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + private static HashSet ExtractScopes(ClaimsPrincipal principal) { var scopes = new HashSet(StringComparer.Ordinal); @@ -108,4 +165,38 @@ internal sealed class StellaOpsScopeAuthorizationHandler : AuthorizationHandler< return scopes; } + + private static IReadOnlyList CombineRequiredScopes( + IReadOnlyList defaultScopes, + IReadOnlyCollection requirementScopes) + { + if ((defaultScopes is null || defaultScopes.Count == 0) && (requirementScopes is null || requirementScopes.Count == 0)) + { + return Array.Empty(); + } + + if (defaultScopes is null || defaultScopes.Count == 0) + { + return requirementScopes is string[] requirementArray + ? requirementArray + : requirementScopes.ToArray(); + } + + var combined = new HashSet(defaultScopes, StringComparer.Ordinal); + + if (requirementScopes is not null) + { + foreach (var scope in requirementScopes) + { + if (!string.IsNullOrWhiteSpace(scope)) + { + combined.Add(scope); + } + } + } + + return combined.Count == defaultScopes.Count && requirementScopes is null + ? defaultScopes + : combined.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs index c65e6867..b2898c66 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs @@ -42,10 +42,38 @@ public class StandardClientProvisioningStoreTests Assert.Equal("bootstrap-client", descriptor!.ClientId); Assert.True(descriptor.Confidential); Assert.Contains("client_credentials", descriptor.AllowedGrantTypes); - Assert.Contains("scopeA", descriptor.AllowedScopes); + Assert.Contains("scopea", descriptor.AllowedScopes); } [Fact] + [Fact] + public async Task CreateOrUpdateAsync_NormalisesTenant() + { + var store = new TrackingClientStore(); + var revocations = new TrackingRevocationStore(); + var provisioning = new StandardClientProvisioningStore("standard", store, revocations, TimeProvider.System); + + var registration = new AuthorityClientRegistration( + clientId: "tenant-client", + confidential: false, + displayName: "Tenant Client", + clientSecret: null, + allowedGrantTypes: new[] { "client_credentials" }, + allowedScopes: new[] { "scopeA" }, + tenant: " Tenant-Alpha " ); + + await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); + + Assert.True(store.Documents.TryGetValue("tenant-client", out var document)); + Assert.NotNull(document); + Assert.Equal("tenant-alpha", document!.Properties[AuthorityClientMetadataKeys.Tenant]); + + var descriptor = await provisioning.FindByClientIdAsync("tenant-client", CancellationToken.None); + Assert.NotNull(descriptor); + Assert.Equal("tenant-alpha", descriptor!.Tenant); + } + + public async Task CreateOrUpdateAsync_StoresAudiences() { var store = new TrackingClientStore(); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj index 84f73910..dba9ce07 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj @@ -1,24 +1,24 @@ - - - net10.0 - preview - enable - enable - true - true - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + true + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs index 8b879c68..7656d338 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs @@ -65,10 +65,20 @@ internal sealed class StandardClientProvisioningStore : IClientProvisioningStore .ToList(); } - foreach (var (key, value) in registration.Properties) - { - document.Properties[key] = value; - } + foreach (var (key, value) in registration.Properties) + { + document.Properties[key] = value; + } + + var normalizedTenant = NormalizeTenant(registration.Tenant); + if (normalizedTenant is not null) + { + document.Properties[AuthorityClientMetadataKeys.Tenant] = normalizedTenant; + } + else + { + document.Properties.Remove(AuthorityClientMetadataKeys.Tenant); + } if (registration.Properties.TryGetValue(AuthorityClientMetadataKeys.SenderConstraint, out var senderConstraintRaw)) { @@ -176,24 +186,27 @@ internal sealed class StandardClientProvisioningStore : IClientProvisioningStore return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); } - private static string JoinValues(IReadOnlyCollection values) - { - if (values is null || values.Count == 0) - { - return string.Empty; - } - - return string.Join( - " ", - values - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Select(static value => value.Trim()) - .OrderBy(static value => value, StringComparer.Ordinal)); - } - - private static AuthorityClientCertificateBinding MapCertificateBinding( - AuthorityClientCertificateBindingRegistration registration, - DateTimeOffset now) + private static string JoinValues(IReadOnlyCollection values) + { + if (values is null || values.Count == 0) + { + return string.Empty; + } + + return string.Join( + " ", + values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .OrderBy(static value => value, StringComparer.Ordinal)); + } + + private static string? NormalizeTenant(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); + + private static AuthorityClientCertificateBinding MapCertificateBinding( + AuthorityClientCertificateBindingRegistration registration, + DateTimeOffset now) { var subjectAlternativeNames = registration.SubjectAlternativeNames.Count == 0 ? new List() diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs index 10c2f18c..7dbd9d9d 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions.Tests/AuthorityClientRegistrationTests.cs @@ -20,12 +20,13 @@ public class AuthorityClientRegistrationTests [Fact] public void WithClientSecret_ReturnsCopy() { - var registration = new AuthorityClientRegistration("cli", false, null, null); + var registration = new AuthorityClientRegistration("cli", false, null, null, tenant: "Tenant-Alpha"); var updated = registration.WithClientSecret("secret"); Assert.Equal("cli", updated.ClientId); Assert.Equal("secret", updated.ClientSecret); Assert.False(updated.Confidential); + Assert.Equal("tenant-alpha", updated.Tenant); } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs index edbdd5fa..e0bff984 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs @@ -6,9 +6,11 @@ namespace StellaOps.Authority.Plugins.Abstractions; public static class AuthorityClientMetadataKeys { public const string AllowedGrantTypes = "allowedGrantTypes"; - public const string AllowedScopes = "allowedScopes"; - public const string Audiences = "audiences"; - public const string RedirectUris = "redirectUris"; - public const string PostLogoutRedirectUris = "postLogoutRedirectUris"; - public const string SenderConstraint = "senderConstraint"; -} + public const string AllowedScopes = "allowedScopes"; + public const string Audiences = "audiences"; + public const string RedirectUris = "redirectUris"; + public const string PostLogoutRedirectUris = "postLogoutRedirectUris"; + public const string SenderConstraint = "senderConstraint"; + public const string Tenant = "tenant"; + public const string ServiceIdentity = "serviceIdentity"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs index e2037fe6..f12c8219 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs @@ -1,800 +1,874 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Claims; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Cryptography.Audit; - -namespace StellaOps.Authority.Plugins.Abstractions; - -/// -/// Describes feature support advertised by an identity provider plugin. -/// -public sealed record AuthorityIdentityProviderCapabilities( - bool SupportsPassword, - bool SupportsMfa, - bool SupportsClientProvisioning) -{ - /// - /// Builds capabilities metadata from a list of capability identifiers. - /// - public static AuthorityIdentityProviderCapabilities FromCapabilities(IEnumerable capabilities) - { - if (capabilities is null) - { - return new AuthorityIdentityProviderCapabilities(false, false, false); - } - - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var entry in capabilities) - { - if (string.IsNullOrWhiteSpace(entry)) - { - continue; - } - - seen.Add(entry.Trim()); - } - - return new AuthorityIdentityProviderCapabilities( - SupportsPassword: seen.Contains(AuthorityPluginCapabilities.Password), - SupportsMfa: seen.Contains(AuthorityPluginCapabilities.Mfa), - SupportsClientProvisioning: seen.Contains(AuthorityPluginCapabilities.ClientProvisioning)); - } -} - -/// -/// Represents a loaded Authority identity provider plugin instance. -/// -public interface IIdentityProviderPlugin -{ - /// - /// Gets the logical name of the plugin instance (matches the manifest key). - /// - string Name { get; } - - /// - /// Gets the plugin type identifier (e.g. standard, ldap). - /// - string Type { get; } - - /// - /// Gets the plugin context comprising the manifest and bound configuration. - /// - AuthorityPluginContext Context { get; } - - /// - /// Gets the credential store responsible for authenticator validation and user provisioning. - /// - IUserCredentialStore Credentials { get; } - - /// - /// Gets the claims enricher applied to issued principals. - /// - IClaimsEnricher ClaimsEnricher { get; } - - /// - /// Gets the optional client provisioning store exposed by the plugin. - /// - IClientProvisioningStore? ClientProvisioning { get; } - - /// - /// Gets the capability metadata advertised by the plugin. - /// - AuthorityIdentityProviderCapabilities Capabilities { get; } - - /// - /// Evaluates the health of the plugin and backing data stores. - /// - /// Token used to cancel the operation. - /// Health result describing the plugin status. - ValueTask CheckHealthAsync(CancellationToken cancellationToken); -} - -/// -/// Supplies operations for validating credentials and managing user records. -/// -public interface IUserCredentialStore -{ - /// - /// Verifies the supplied username/password combination. - /// - ValueTask VerifyPasswordAsync( - string username, - string password, - CancellationToken cancellationToken); - - /// - /// Creates or updates a user record based on the supplied registration data. - /// - ValueTask> UpsertUserAsync( - AuthorityUserRegistration registration, - CancellationToken cancellationToken); - - /// - /// Attempts to resolve a user descriptor by its canonical subject identifier. - /// - ValueTask FindBySubjectAsync( - string subjectId, - CancellationToken cancellationToken); -} - -/// -/// Enriches issued principals with additional claims based on plugin-specific rules. -/// -public interface IClaimsEnricher -{ - /// - /// Adds or adjusts claims on the provided identity. - /// - ValueTask EnrichAsync( - ClaimsIdentity identity, - AuthorityClaimsEnrichmentContext context, - CancellationToken cancellationToken); -} - -/// -/// Manages client (machine-to-machine) provisioning for Authority. -/// -public interface IClientProvisioningStore -{ - /// - /// Creates or updates a client registration. - /// - ValueTask> CreateOrUpdateAsync( - AuthorityClientRegistration registration, - CancellationToken cancellationToken); - - /// - /// Attempts to resolve a client descriptor by its identifier. - /// - ValueTask FindByClientIdAsync( - string clientId, - CancellationToken cancellationToken); - - /// - /// Removes a client registration. - /// - ValueTask DeleteAsync( - string clientId, - CancellationToken cancellationToken); -} - -/// -/// Represents the health state of a plugin or backing store. -/// -public enum AuthorityPluginHealthStatus -{ - /// - /// Plugin is healthy and operational. - /// - Healthy, - - /// - /// Plugin is degraded but still usable (e.g. transient connectivity issues). - /// - Degraded, - - /// - /// Plugin is unavailable and cannot service requests. - /// - Unavailable -} - -/// -/// Result of a plugin health probe. -/// -public sealed record AuthorityPluginHealthResult -{ - private AuthorityPluginHealthResult( - AuthorityPluginHealthStatus status, - string? message, - IReadOnlyDictionary details) - { - Status = status; - Message = message; - Details = details; - } - - /// - /// Gets the overall status of the plugin. - /// - public AuthorityPluginHealthStatus Status { get; } - - /// - /// Gets an optional human-readable status description. - /// - public string? Message { get; } - - /// - /// Gets optional structured details for diagnostics. - /// - public IReadOnlyDictionary Details { get; } - - /// - /// Creates a healthy result. - /// - public static AuthorityPluginHealthResult Healthy( - string? message = null, - IReadOnlyDictionary? details = null) - => new(AuthorityPluginHealthStatus.Healthy, message, details ?? EmptyDetails); - - /// - /// Creates a degraded result. - /// - public static AuthorityPluginHealthResult Degraded( - string? message = null, - IReadOnlyDictionary? details = null) - => new(AuthorityPluginHealthStatus.Degraded, message, details ?? EmptyDetails); - - /// - /// Creates an unavailable result. - /// - public static AuthorityPluginHealthResult Unavailable( - string? message = null, - IReadOnlyDictionary? details = null) - => new(AuthorityPluginHealthStatus.Unavailable, message, details ?? EmptyDetails); - - private static readonly IReadOnlyDictionary EmptyDetails = - new Dictionary(StringComparer.OrdinalIgnoreCase); -} - -/// -/// Describes a canonical Authority user surfaced by a plugin. -/// -public sealed record AuthorityUserDescriptor -{ - /// - /// Initialises a new user descriptor. - /// - public AuthorityUserDescriptor( - string subjectId, - string username, - string? displayName, - bool requiresPasswordReset, - IReadOnlyCollection? roles = null, - IReadOnlyDictionary? attributes = null) - { - SubjectId = ValidateRequired(subjectId, nameof(subjectId)); - Username = ValidateRequired(username, nameof(username)); - DisplayName = displayName; - RequiresPasswordReset = requiresPasswordReset; - Roles = roles is null ? Array.Empty() : roles.ToArray(); - Attributes = attributes is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); - } - - /// - /// Stable subject identifier for token issuance. - /// - public string SubjectId { get; } - - /// - /// Canonical username (case-normalised). - /// - public string Username { get; } - - /// - /// Optional human-friendly display name. - /// - public string? DisplayName { get; } - - /// - /// Indicates whether the user must reset their password. - /// - public bool RequiresPasswordReset { get; } - - /// - /// Collection of role identifiers associated with the user. - /// - public IReadOnlyCollection Roles { get; } - - /// - /// Arbitrary plugin-defined attributes (used by claims enricher). - /// - public IReadOnlyDictionary Attributes { get; } - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} - -/// -/// Outcome of a credential verification attempt. -/// -public sealed record AuthorityCredentialVerificationResult -{ - private AuthorityCredentialVerificationResult( - bool succeeded, - AuthorityUserDescriptor? user, - AuthorityCredentialFailureCode? failureCode, - string? message, - TimeSpan? retryAfter, - IReadOnlyList auditProperties) - { - Succeeded = succeeded; - User = user; - FailureCode = failureCode; - Message = message; - RetryAfter = retryAfter; - AuditProperties = auditProperties ?? Array.Empty(); - } - - /// - /// Indicates whether the verification succeeded. - /// - public bool Succeeded { get; } - - /// - /// Resolved user descriptor when successful. - /// - public AuthorityUserDescriptor? User { get; } - - /// - /// Failure classification when unsuccessful. - /// - public AuthorityCredentialFailureCode? FailureCode { get; } - - /// - /// Optional message describing the outcome. - /// - public string? Message { get; } - - /// - /// Optional suggested retry interval (e.g. for lockouts). - /// - public TimeSpan? RetryAfter { get; } - - /// - /// Additional audit properties emitted by the credential store. - /// - public IReadOnlyList AuditProperties { get; } - - /// - /// Builds a successful verification result. - /// - public static AuthorityCredentialVerificationResult Success( - AuthorityUserDescriptor user, - string? message = null, - IReadOnlyList? auditProperties = null) - => new(true, user ?? throw new ArgumentNullException(nameof(user)), null, message, null, auditProperties ?? Array.Empty()); - - /// - /// Builds a failed verification result. - /// - public static AuthorityCredentialVerificationResult Failure( - AuthorityCredentialFailureCode failureCode, - string? message = null, - TimeSpan? retryAfter = null, - IReadOnlyList? auditProperties = null) - => new(false, null, failureCode, message, retryAfter, auditProperties ?? Array.Empty()); -} - -/// -/// Classifies credential verification failures. -/// -public enum AuthorityCredentialFailureCode -{ - /// - /// Username/password combination is invalid. - /// - InvalidCredentials, - - /// - /// Account is locked out (retry after a specified duration). - /// - LockedOut, - - /// - /// Password has expired and must be reset. - /// - PasswordExpired, - - /// - /// User must reset password before proceeding. - /// - RequiresPasswordReset, - - /// - /// Additional multi-factor authentication is required. - /// - RequiresMfa, - - /// - /// Unexpected failure occurred (see message for details). - /// - UnknownError -} - -/// -/// Represents a user provisioning request. -/// -public sealed record AuthorityUserRegistration -{ - /// - /// Initialises a new registration. - /// - public AuthorityUserRegistration( - string username, - string? password, - string? displayName, - string? email, - bool requirePasswordReset, - IReadOnlyCollection? roles = null, - IReadOnlyDictionary? attributes = null) - { - Username = ValidateRequired(username, nameof(username)); - Password = password; - DisplayName = displayName; - Email = email; - RequirePasswordReset = requirePasswordReset; - Roles = roles is null ? Array.Empty() : roles.ToArray(); - Attributes = attributes is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); - } - - /// - /// Canonical username (unique). - /// - public string Username { get; } - - /// - /// Optional raw password (hashed by plugin). - /// - public string? Password { get; init; } - - /// - /// Optional human-friendly display name. - /// - public string? DisplayName { get; } - - /// - /// Optional contact email. - /// - public string? Email { get; } - - /// - /// Indicates whether the user must reset their password at next login. - /// - public bool RequirePasswordReset { get; } - - /// - /// Associated roles. - /// - public IReadOnlyCollection Roles { get; } - - /// - /// Plugin-defined attributes. - /// - public IReadOnlyDictionary Attributes { get; } - - /// - /// Creates a copy with the provided password while preserving other fields. - /// - public AuthorityUserRegistration WithPassword(string? password) - => new(Username, password, DisplayName, Email, RequirePasswordReset, Roles, Attributes); - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} - -/// -/// Generic operation result utilised by plugins. -/// -public sealed record AuthorityPluginOperationResult -{ - private AuthorityPluginOperationResult(bool succeeded, string? errorCode, string? message) - { - Succeeded = succeeded; - ErrorCode = errorCode; - Message = message; - } - - /// - /// Indicates whether the operation succeeded. - /// - public bool Succeeded { get; } - - /// - /// Machine-readable error code (populated on failure). - /// - public string? ErrorCode { get; } - - /// - /// Optional human-readable message. - /// - public string? Message { get; } - - /// - /// Returns a successful result. - /// - public static AuthorityPluginOperationResult Success(string? message = null) - => new(true, null, message); - - /// - /// Returns a failed result with the supplied error code. - /// - public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) - => new(false, ValidateErrorCode(errorCode), message); - - internal static string ValidateErrorCode(string errorCode) - => string.IsNullOrWhiteSpace(errorCode) - ? throw new ArgumentException("Error code is required for failures.", nameof(errorCode)) - : errorCode; -} - -/// -/// Generic operation result that returns a value. -/// -public sealed record AuthorityPluginOperationResult -{ - private AuthorityPluginOperationResult( - bool succeeded, - TValue? value, - string? errorCode, - string? message) - { - Succeeded = succeeded; - Value = value; - ErrorCode = errorCode; - Message = message; - } - - /// - /// Indicates whether the operation succeeded. - /// - public bool Succeeded { get; } - - /// - /// Returned value when successful. - /// - public TValue? Value { get; } - - /// - /// Machine-readable error code (on failure). - /// - public string? ErrorCode { get; } - - /// - /// Optional human-readable message. - /// - public string? Message { get; } - - /// - /// Returns a successful result with the provided value. - /// - public static AuthorityPluginOperationResult Success(TValue value, string? message = null) - => new(true, value, null, message); - - /// - /// Returns a successful result without a value (defaults to default). - /// - public static AuthorityPluginOperationResult Success(string? message = null) - => new(true, default, null, message); - - /// - /// Returns a failed result with the supplied error code. - /// - public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) - => new(false, default, AuthorityPluginOperationResult.ValidateErrorCode(errorCode), message); -} - -/// -/// Context supplied to claims enrichment routines. -/// -public sealed class AuthorityClaimsEnrichmentContext -{ - private readonly Dictionary items; - - /// - /// Initialises a new context instance. - /// - public AuthorityClaimsEnrichmentContext( - AuthorityPluginContext plugin, - AuthorityUserDescriptor? user, - AuthorityClientDescriptor? client) - { - Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin)); - User = user; - Client = client; - items = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - - /// - /// Gets the plugin context associated with the principal. - /// - public AuthorityPluginContext Plugin { get; } - - /// - /// Gets the user descriptor when available. - /// - public AuthorityUserDescriptor? User { get; } - - /// - /// Gets the client descriptor when available. - /// - public AuthorityClientDescriptor? Client { get; } - - /// - /// Extensible bag for plugin-specific data passed between enrichment stages. - /// - public IDictionary Items => items; -} - -/// -/// Represents a registered OAuth/OpenID client. -/// -public sealed record AuthorityClientDescriptor -{ - public AuthorityClientDescriptor( - string clientId, - string? displayName, - bool confidential, - IReadOnlyCollection? allowedGrantTypes = null, - IReadOnlyCollection? allowedScopes = null, - IReadOnlyCollection? allowedAudiences = null, - IReadOnlyCollection? redirectUris = null, - IReadOnlyCollection? postLogoutRedirectUris = null, - IReadOnlyDictionary? properties = null) - { - ClientId = ValidateRequired(clientId, nameof(clientId)); - DisplayName = displayName; - Confidential = confidential; - AllowedGrantTypes = Normalize(allowedGrantTypes); - AllowedScopes = Normalize(allowedScopes); - AllowedAudiences = Normalize(allowedAudiences); - RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); - PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); - Properties = properties is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); - } - - public string ClientId { get; } - public string? DisplayName { get; } - public bool Confidential { get; } - public IReadOnlyCollection AllowedGrantTypes { get; } - public IReadOnlyCollection AllowedScopes { get; } - public IReadOnlyCollection AllowedAudiences { get; } - public IReadOnlyCollection RedirectUris { get; } - public IReadOnlyCollection PostLogoutRedirectUris { get; } - public IReadOnlyDictionary Properties { get; } - - private static IReadOnlyCollection Normalize(IReadOnlyCollection? values) - => values is null || values.Count == 0 - ? Array.Empty() - : values - .Where(value => !string.IsNullOrWhiteSpace(value)) - .Select(value => value.Trim()) - .Distinct(StringComparer.Ordinal) - .ToArray(); - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} - -public sealed record AuthorityClientCertificateBindingRegistration -{ - public AuthorityClientCertificateBindingRegistration( - string thumbprint, - string? serialNumber = null, - string? subject = null, - string? issuer = null, - IReadOnlyCollection? subjectAlternativeNames = null, - DateTimeOffset? notBefore = null, - DateTimeOffset? notAfter = null, - string? label = null) - { - Thumbprint = NormalizeThumbprint(thumbprint); - SerialNumber = Normalize(serialNumber); - Subject = Normalize(subject); - Issuer = Normalize(issuer); - SubjectAlternativeNames = subjectAlternativeNames is null || subjectAlternativeNames.Count == 0 - ? Array.Empty() - : subjectAlternativeNames - .Where(value => !string.IsNullOrWhiteSpace(value)) - .Select(value => value.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - NotBefore = notBefore; - NotAfter = notAfter; - Label = Normalize(label); - } - - public string Thumbprint { get; } - public string? SerialNumber { get; } - public string? Subject { get; } - public string? Issuer { get; } - public IReadOnlyCollection SubjectAlternativeNames { get; } - public DateTimeOffset? NotBefore { get; } - public DateTimeOffset? NotAfter { get; } - public string? Label { get; } - - private static string NormalizeThumbprint(string value) - { - if (string.IsNullOrWhiteSpace(value)) - { - throw new ArgumentException("Thumbprint is required.", nameof(value)); - } - - return value - .Replace(":", string.Empty, StringComparison.Ordinal) - .Replace(" ", string.Empty, StringComparison.Ordinal) - .ToUpperInvariant(); - } - - private static string? Normalize(string? value) - => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); -} - -public sealed record AuthorityClientRegistration -{ - public AuthorityClientRegistration( - string clientId, - bool confidential, - string? displayName, - string? clientSecret, - IReadOnlyCollection? allowedGrantTypes = null, - IReadOnlyCollection? allowedScopes = null, - IReadOnlyCollection? allowedAudiences = null, - IReadOnlyCollection? redirectUris = null, - IReadOnlyCollection? postLogoutRedirectUris = null, - IReadOnlyDictionary? properties = null, - IReadOnlyCollection? certificateBindings = null) - { - ClientId = ValidateRequired(clientId, nameof(clientId)); - Confidential = confidential; - DisplayName = displayName; - ClientSecret = confidential - ? ValidateRequired(clientSecret ?? string.Empty, nameof(clientSecret)) - : clientSecret; - AllowedGrantTypes = Normalize(allowedGrantTypes); - AllowedScopes = Normalize(allowedScopes); - AllowedAudiences = Normalize(allowedAudiences); - RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); - PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); - Properties = properties is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); - CertificateBindings = certificateBindings is null - ? Array.Empty() - : certificateBindings.ToArray(); - } - - public string ClientId { get; } - public bool Confidential { get; } - public string? DisplayName { get; } - public string? ClientSecret { get; init; } - public IReadOnlyCollection AllowedGrantTypes { get; } - public IReadOnlyCollection AllowedScopes { get; } - public IReadOnlyCollection AllowedAudiences { get; } - public IReadOnlyCollection RedirectUris { get; } - public IReadOnlyCollection PostLogoutRedirectUris { get; } - public IReadOnlyDictionary Properties { get; } - public IReadOnlyCollection CertificateBindings { get; } - - public AuthorityClientRegistration WithClientSecret(string? clientSecret) - => new(ClientId, Confidential, DisplayName, clientSecret, AllowedGrantTypes, AllowedScopes, AllowedAudiences, RedirectUris, PostLogoutRedirectUris, Properties, CertificateBindings); - - private static IReadOnlyCollection Normalize(IReadOnlyCollection? values) - => values is null || values.Count == 0 - ? Array.Empty() - : values - .Where(value => !string.IsNullOrWhiteSpace(value)) - .Select(value => value.Trim()) - .Distinct(StringComparer.Ordinal) - .ToArray(); - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cryptography.Audit; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Describes feature support advertised by an identity provider plugin. +/// +public sealed record AuthorityIdentityProviderCapabilities( + bool SupportsPassword, + bool SupportsMfa, + bool SupportsClientProvisioning) +{ + /// + /// Builds capabilities metadata from a list of capability identifiers. + /// + public static AuthorityIdentityProviderCapabilities FromCapabilities(IEnumerable capabilities) + { + if (capabilities is null) + { + return new AuthorityIdentityProviderCapabilities(false, false, false); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in capabilities) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + seen.Add(entry.Trim()); + } + + return new AuthorityIdentityProviderCapabilities( + SupportsPassword: seen.Contains(AuthorityPluginCapabilities.Password), + SupportsMfa: seen.Contains(AuthorityPluginCapabilities.Mfa), + SupportsClientProvisioning: seen.Contains(AuthorityPluginCapabilities.ClientProvisioning)); + } +} + +/// +/// Represents a loaded Authority identity provider plugin instance. +/// +public interface IIdentityProviderPlugin +{ + /// + /// Gets the logical name of the plugin instance (matches the manifest key). + /// + string Name { get; } + + /// + /// Gets the plugin type identifier (e.g. standard, ldap). + /// + string Type { get; } + + /// + /// Gets the plugin context comprising the manifest and bound configuration. + /// + AuthorityPluginContext Context { get; } + + /// + /// Gets the credential store responsible for authenticator validation and user provisioning. + /// + IUserCredentialStore Credentials { get; } + + /// + /// Gets the claims enricher applied to issued principals. + /// + IClaimsEnricher ClaimsEnricher { get; } + + /// + /// Gets the optional client provisioning store exposed by the plugin. + /// + IClientProvisioningStore? ClientProvisioning { get; } + + /// + /// Gets the capability metadata advertised by the plugin. + /// + AuthorityIdentityProviderCapabilities Capabilities { get; } + + /// + /// Evaluates the health of the plugin and backing data stores. + /// + /// Token used to cancel the operation. + /// Health result describing the plugin status. + ValueTask CheckHealthAsync(CancellationToken cancellationToken); +} + +/// +/// Supplies operations for validating credentials and managing user records. +/// +public interface IUserCredentialStore +{ + /// + /// Verifies the supplied username/password combination. + /// + ValueTask VerifyPasswordAsync( + string username, + string password, + CancellationToken cancellationToken); + + /// + /// Creates or updates a user record based on the supplied registration data. + /// + ValueTask> UpsertUserAsync( + AuthorityUserRegistration registration, + CancellationToken cancellationToken); + + /// + /// Attempts to resolve a user descriptor by its canonical subject identifier. + /// + ValueTask FindBySubjectAsync( + string subjectId, + CancellationToken cancellationToken); +} + +/// +/// Enriches issued principals with additional claims based on plugin-specific rules. +/// +public interface IClaimsEnricher +{ + /// + /// Adds or adjusts claims on the provided identity. + /// + ValueTask EnrichAsync( + ClaimsIdentity identity, + AuthorityClaimsEnrichmentContext context, + CancellationToken cancellationToken); +} + +/// +/// Manages client (machine-to-machine) provisioning for Authority. +/// +public interface IClientProvisioningStore +{ + /// + /// Creates or updates a client registration. + /// + ValueTask> CreateOrUpdateAsync( + AuthorityClientRegistration registration, + CancellationToken cancellationToken); + + /// + /// Attempts to resolve a client descriptor by its identifier. + /// + ValueTask FindByClientIdAsync( + string clientId, + CancellationToken cancellationToken); + + /// + /// Removes a client registration. + /// + ValueTask DeleteAsync( + string clientId, + CancellationToken cancellationToken); +} + +/// +/// Represents the health state of a plugin or backing store. +/// +public enum AuthorityPluginHealthStatus +{ + /// + /// Plugin is healthy and operational. + /// + Healthy, + + /// + /// Plugin is degraded but still usable (e.g. transient connectivity issues). + /// + Degraded, + + /// + /// Plugin is unavailable and cannot service requests. + /// + Unavailable +} + +/// +/// Result of a plugin health probe. +/// +public sealed record AuthorityPluginHealthResult +{ + private AuthorityPluginHealthResult( + AuthorityPluginHealthStatus status, + string? message, + IReadOnlyDictionary details) + { + Status = status; + Message = message; + Details = details; + } + + /// + /// Gets the overall status of the plugin. + /// + public AuthorityPluginHealthStatus Status { get; } + + /// + /// Gets an optional human-readable status description. + /// + public string? Message { get; } + + /// + /// Gets optional structured details for diagnostics. + /// + public IReadOnlyDictionary Details { get; } + + /// + /// Creates a healthy result. + /// + public static AuthorityPluginHealthResult Healthy( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Healthy, message, details ?? EmptyDetails); + + /// + /// Creates a degraded result. + /// + public static AuthorityPluginHealthResult Degraded( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Degraded, message, details ?? EmptyDetails); + + /// + /// Creates an unavailable result. + /// + public static AuthorityPluginHealthResult Unavailable( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Unavailable, message, details ?? EmptyDetails); + + private static readonly IReadOnlyDictionary EmptyDetails = + new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +/// +/// Describes a canonical Authority user surfaced by a plugin. +/// +public sealed record AuthorityUserDescriptor +{ + /// + /// Initialises a new user descriptor. + /// + public AuthorityUserDescriptor( + string subjectId, + string username, + string? displayName, + bool requiresPasswordReset, + IReadOnlyCollection? roles = null, + IReadOnlyDictionary? attributes = null) + { + SubjectId = ValidateRequired(subjectId, nameof(subjectId)); + Username = ValidateRequired(username, nameof(username)); + DisplayName = displayName; + RequiresPasswordReset = requiresPasswordReset; + Roles = roles is null ? Array.Empty() : roles.ToArray(); + Attributes = attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Stable subject identifier for token issuance. + /// + public string SubjectId { get; } + + /// + /// Canonical username (case-normalised). + /// + public string Username { get; } + + /// + /// Optional human-friendly display name. + /// + public string? DisplayName { get; } + + /// + /// Indicates whether the user must reset their password. + /// + public bool RequiresPasswordReset { get; } + + /// + /// Collection of role identifiers associated with the user. + /// + public IReadOnlyCollection Roles { get; } + + /// + /// Arbitrary plugin-defined attributes (used by claims enricher). + /// + public IReadOnlyDictionary Attributes { get; } + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Outcome of a credential verification attempt. +/// +public sealed record AuthorityCredentialVerificationResult +{ + private AuthorityCredentialVerificationResult( + bool succeeded, + AuthorityUserDescriptor? user, + AuthorityCredentialFailureCode? failureCode, + string? message, + TimeSpan? retryAfter, + IReadOnlyList auditProperties) + { + Succeeded = succeeded; + User = user; + FailureCode = failureCode; + Message = message; + RetryAfter = retryAfter; + AuditProperties = auditProperties ?? Array.Empty(); + } + + /// + /// Indicates whether the verification succeeded. + /// + public bool Succeeded { get; } + + /// + /// Resolved user descriptor when successful. + /// + public AuthorityUserDescriptor? User { get; } + + /// + /// Failure classification when unsuccessful. + /// + public AuthorityCredentialFailureCode? FailureCode { get; } + + /// + /// Optional message describing the outcome. + /// + public string? Message { get; } + + /// + /// Optional suggested retry interval (e.g. for lockouts). + /// + public TimeSpan? RetryAfter { get; } + + /// + /// Additional audit properties emitted by the credential store. + /// + public IReadOnlyList AuditProperties { get; } + + /// + /// Builds a successful verification result. + /// + public static AuthorityCredentialVerificationResult Success( + AuthorityUserDescriptor user, + string? message = null, + IReadOnlyList? auditProperties = null) + => new(true, user ?? throw new ArgumentNullException(nameof(user)), null, message, null, auditProperties ?? Array.Empty()); + + /// + /// Builds a failed verification result. + /// + public static AuthorityCredentialVerificationResult Failure( + AuthorityCredentialFailureCode failureCode, + string? message = null, + TimeSpan? retryAfter = null, + IReadOnlyList? auditProperties = null) + => new(false, null, failureCode, message, retryAfter, auditProperties ?? Array.Empty()); +} + +/// +/// Classifies credential verification failures. +/// +public enum AuthorityCredentialFailureCode +{ + /// + /// Username/password combination is invalid. + /// + InvalidCredentials, + + /// + /// Account is locked out (retry after a specified duration). + /// + LockedOut, + + /// + /// Password has expired and must be reset. + /// + PasswordExpired, + + /// + /// User must reset password before proceeding. + /// + RequiresPasswordReset, + + /// + /// Additional multi-factor authentication is required. + /// + RequiresMfa, + + /// + /// Unexpected failure occurred (see message for details). + /// + UnknownError +} + +/// +/// Represents a user provisioning request. +/// +public sealed record AuthorityUserRegistration +{ + /// + /// Initialises a new registration. + /// + public AuthorityUserRegistration( + string username, + string? password, + string? displayName, + string? email, + bool requirePasswordReset, + IReadOnlyCollection? roles = null, + IReadOnlyDictionary? attributes = null) + { + Username = ValidateRequired(username, nameof(username)); + Password = password; + DisplayName = displayName; + Email = email; + RequirePasswordReset = requirePasswordReset; + Roles = roles is null ? Array.Empty() : roles.ToArray(); + Attributes = attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Canonical username (unique). + /// + public string Username { get; } + + /// + /// Optional raw password (hashed by plugin). + /// + public string? Password { get; init; } + + /// + /// Optional human-friendly display name. + /// + public string? DisplayName { get; } + + /// + /// Optional contact email. + /// + public string? Email { get; } + + /// + /// Indicates whether the user must reset their password at next login. + /// + public bool RequirePasswordReset { get; } + + /// + /// Associated roles. + /// + public IReadOnlyCollection Roles { get; } + + /// + /// Plugin-defined attributes. + /// + public IReadOnlyDictionary Attributes { get; } + + /// + /// Creates a copy with the provided password while preserving other fields. + /// + public AuthorityUserRegistration WithPassword(string? password) + => new(Username, password, DisplayName, Email, RequirePasswordReset, Roles, Attributes); + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Generic operation result utilised by plugins. +/// +public sealed record AuthorityPluginOperationResult +{ + private AuthorityPluginOperationResult(bool succeeded, string? errorCode, string? message) + { + Succeeded = succeeded; + ErrorCode = errorCode; + Message = message; + } + + /// + /// Indicates whether the operation succeeded. + /// + public bool Succeeded { get; } + + /// + /// Machine-readable error code (populated on failure). + /// + public string? ErrorCode { get; } + + /// + /// Optional human-readable message. + /// + public string? Message { get; } + + /// + /// Returns a successful result. + /// + public static AuthorityPluginOperationResult Success(string? message = null) + => new(true, null, message); + + /// + /// Returns a failed result with the supplied error code. + /// + public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) + => new(false, ValidateErrorCode(errorCode), message); + + internal static string ValidateErrorCode(string errorCode) + => string.IsNullOrWhiteSpace(errorCode) + ? throw new ArgumentException("Error code is required for failures.", nameof(errorCode)) + : errorCode; +} + +/// +/// Generic operation result that returns a value. +/// +public sealed record AuthorityPluginOperationResult +{ + private AuthorityPluginOperationResult( + bool succeeded, + TValue? value, + string? errorCode, + string? message) + { + Succeeded = succeeded; + Value = value; + ErrorCode = errorCode; + Message = message; + } + + /// + /// Indicates whether the operation succeeded. + /// + public bool Succeeded { get; } + + /// + /// Returned value when successful. + /// + public TValue? Value { get; } + + /// + /// Machine-readable error code (on failure). + /// + public string? ErrorCode { get; } + + /// + /// Optional human-readable message. + /// + public string? Message { get; } + + /// + /// Returns a successful result with the provided value. + /// + public static AuthorityPluginOperationResult Success(TValue value, string? message = null) + => new(true, value, null, message); + + /// + /// Returns a successful result without a value (defaults to default). + /// + public static AuthorityPluginOperationResult Success(string? message = null) + => new(true, default, null, message); + + /// + /// Returns a failed result with the supplied error code. + /// + public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) + => new(false, default, AuthorityPluginOperationResult.ValidateErrorCode(errorCode), message); +} + +/// +/// Context supplied to claims enrichment routines. +/// +public sealed class AuthorityClaimsEnrichmentContext +{ + private readonly Dictionary items; + + /// + /// Initialises a new context instance. + /// + public AuthorityClaimsEnrichmentContext( + AuthorityPluginContext plugin, + AuthorityUserDescriptor? user, + AuthorityClientDescriptor? client) + { + Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin)); + User = user; + Client = client; + items = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + /// Gets the plugin context associated with the principal. + /// + public AuthorityPluginContext Plugin { get; } + + /// + /// Gets the user descriptor when available. + /// + public AuthorityUserDescriptor? User { get; } + + /// + /// Gets the client descriptor when available. + /// + public AuthorityClientDescriptor? Client { get; } + + /// + /// Extensible bag for plugin-specific data passed between enrichment stages. + /// + public IDictionary Items => items; +} + +/// +/// Represents a registered OAuth/OpenID client. +/// +public sealed record AuthorityClientDescriptor +{ + public AuthorityClientDescriptor( + string clientId, + string? displayName, + bool confidential, + IReadOnlyCollection? allowedGrantTypes = null, + IReadOnlyCollection? allowedScopes = null, + IReadOnlyCollection? allowedAudiences = null, + IReadOnlyCollection? redirectUris = null, + IReadOnlyCollection? postLogoutRedirectUris = null, + IReadOnlyDictionary? properties = null) + { + ClientId = ValidateRequired(clientId, nameof(clientId)); + DisplayName = displayName; + Confidential = confidential; + AllowedGrantTypes = Normalize(allowedGrantTypes); + AllowedScopes = NormalizeScopes(allowedScopes); + AllowedAudiences = Normalize(allowedAudiences); + RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); + PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); + Properties = properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + Tenant = Properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var tenantValue) + ? AuthorityClientRegistration.NormalizeTenantValue(tenantValue) + : null; + } + + public string ClientId { get; } + public string? DisplayName { get; } + public bool Confidential { get; } + public IReadOnlyCollection AllowedGrantTypes { get; } + public IReadOnlyCollection AllowedScopes { get; } + public IReadOnlyCollection AllowedAudiences { get; } + public IReadOnlyCollection RedirectUris { get; } + public IReadOnlyCollection PostLogoutRedirectUris { get; } + public string? Tenant { get; } + public IReadOnlyDictionary Properties { get; } + + private static IReadOnlyCollection Normalize(IReadOnlyCollection? values) + => values is null || values.Count == 0 + ? Array.Empty() + : values + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + private static IReadOnlyCollection NormalizeScopes(IReadOnlyCollection? values) + { + if (values is null || values.Count == 0) + { + return Array.Empty(); + } + + var unique = new HashSet(StringComparer.Ordinal); + + foreach (var value in values) + { + var normalized = StellaOpsScopes.Normalize(value); + if (normalized is null) + { + continue; + } + + unique.Add(normalized); + } + + if (unique.Count == 0) + { + return Array.Empty(); + } + + return unique.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + } + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +public sealed record AuthorityClientCertificateBindingRegistration +{ + public AuthorityClientCertificateBindingRegistration( + string thumbprint, + string? serialNumber = null, + string? subject = null, + string? issuer = null, + IReadOnlyCollection? subjectAlternativeNames = null, + DateTimeOffset? notBefore = null, + DateTimeOffset? notAfter = null, + string? label = null) + { + Thumbprint = NormalizeThumbprint(thumbprint); + SerialNumber = Normalize(serialNumber); + Subject = Normalize(subject); + Issuer = Normalize(issuer); + SubjectAlternativeNames = subjectAlternativeNames is null || subjectAlternativeNames.Count == 0 + ? Array.Empty() + : subjectAlternativeNames + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + NotBefore = notBefore; + NotAfter = notAfter; + Label = Normalize(label); + } + + public string Thumbprint { get; } + public string? SerialNumber { get; } + public string? Subject { get; } + public string? Issuer { get; } + public IReadOnlyCollection SubjectAlternativeNames { get; } + public DateTimeOffset? NotBefore { get; } + public DateTimeOffset? NotAfter { get; } + public string? Label { get; } + + private static string NormalizeThumbprint(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Thumbprint is required.", nameof(value)); + } + + return value + .Replace(":", string.Empty, StringComparison.Ordinal) + .Replace(" ", string.Empty, StringComparison.Ordinal) + .ToUpperInvariant(); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); +} + +public sealed record AuthorityClientRegistration +{ + public AuthorityClientRegistration( + string clientId, + bool confidential, + string? displayName, + string? clientSecret, + IReadOnlyCollection? allowedGrantTypes = null, + IReadOnlyCollection? allowedScopes = null, + IReadOnlyCollection? allowedAudiences = null, + IReadOnlyCollection? redirectUris = null, + IReadOnlyCollection? postLogoutRedirectUris = null, + string? tenant = null, + IReadOnlyDictionary? properties = null, + IReadOnlyCollection? certificateBindings = null) + { + ClientId = ValidateRequired(clientId, nameof(clientId)); + Confidential = confidential; + DisplayName = displayName; + ClientSecret = confidential + ? ValidateRequired(clientSecret ?? string.Empty, nameof(clientSecret)) + : clientSecret; + AllowedGrantTypes = Normalize(allowedGrantTypes); + AllowedScopes = NormalizeScopes(allowedScopes); + AllowedAudiences = Normalize(allowedAudiences); + RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); + PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); + Tenant = NormalizeTenantValue(tenant); + Properties = properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + CertificateBindings = certificateBindings is null + ? Array.Empty() + : certificateBindings.ToArray(); + } + + public string ClientId { get; } + public bool Confidential { get; } + public string? DisplayName { get; } + public string? ClientSecret { get; init; } + public IReadOnlyCollection AllowedGrantTypes { get; } + public IReadOnlyCollection AllowedScopes { get; } + public IReadOnlyCollection AllowedAudiences { get; } + public IReadOnlyCollection RedirectUris { get; } + public IReadOnlyCollection PostLogoutRedirectUris { get; } + public string? Tenant { get; } + public IReadOnlyDictionary Properties { get; } + public IReadOnlyCollection CertificateBindings { get; } + + public AuthorityClientRegistration WithClientSecret(string? clientSecret) + => new(ClientId, Confidential, DisplayName, clientSecret, AllowedGrantTypes, AllowedScopes, AllowedAudiences, RedirectUris, PostLogoutRedirectUris, Tenant, Properties, CertificateBindings); + + private static IReadOnlyCollection Normalize(IReadOnlyCollection? values) + => values is null || values.Count == 0 + ? Array.Empty() + : values + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + private static IReadOnlyCollection NormalizeScopes(IReadOnlyCollection? values) + { + if (values is null || values.Count == 0) + { + return Array.Empty(); + } + + var unique = new HashSet(StringComparer.Ordinal); + + foreach (var value in values) + { + var normalized = StellaOpsScopes.Normalize(value); + if (normalized is null) + { + continue; + } + + unique.Add(normalized); + } + + if (unique.Count == 0) + { + return Array.Empty(); + } + + return unique.OrderBy(static scope => scope, StringComparer.Ordinal).ToArray(); + } + + internal static string? NormalizeTenantValue(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim().ToLowerInvariant(); + } + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj index 6be9ac0b..7b6eadeb 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/StellaOps.Authority.Plugins.Abstractions.csproj @@ -7,11 +7,12 @@ true - - - + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs index c68e38ac..e9fc79e8 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityLoginAttemptDocument.cs @@ -35,6 +35,10 @@ public sealed class AuthorityLoginAttemptDocument [BsonIgnoreIfNull] public string? ClientId { get; set; } + [BsonElement("tenant")] + [BsonIgnoreIfNull] + public string? Tenant { get; set; } + [BsonElement("plugin")] [BsonIgnoreIfNull] public string? Plugin { get; set; } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs index 79df9ac0..2234f718 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs @@ -74,6 +74,9 @@ public sealed class AuthorityTokenDocument [BsonIgnoreIfNull] public string? SenderNonce { get; set; } + [BsonElement("tenant")] + [BsonIgnoreIfNull] + public string? Tenant { get; set; } [BsonElement("devices")] [BsonIgnoreIfNull] diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs index c4a5ae90..1a392025 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityLoginAttemptCollectionInitializer.cs @@ -22,7 +22,12 @@ internal sealed class AuthorityLoginAttemptCollectionInitializer : IAuthorityCol new CreateIndexModel( Builders.IndexKeys .Ascending(a => a.CorrelationId), - new CreateIndexOptions { Name = "login_attempt_correlation", Sparse = true }) + new CreateIndexOptions { Name = "login_attempt_correlation", Sparse = true }), + new CreateIndexModel( + Builders.IndexKeys + .Ascending(a => a.Tenant) + .Descending(a => a.OccurredAt), + new CreateIndexOptions { Name = "login_attempt_tenant_time", Sparse = true }) }; await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj index 344af1c5..f06e4c35 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs index e3ae26f1..10d71e63 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs @@ -101,6 +101,7 @@ public class ClientCredentialsHandlersTests await handler.HandleAsync(context); Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}"); + Assert.False(context.Transaction.Properties.ContainsKey(AuthorityOpenIddictConstants.ClientTenantProperty)); Assert.Same(clientDocument, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty]); var grantedScopes = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]); @@ -108,6 +109,323 @@ public class ClientCredentialsHandlersTests Assert.Equal(clientDocument.Plugin, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty]); } + [Fact] + public async Task ValidateClientCredentials_Allows_NewIngestionScopes() + { + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "advisory:ingest advisory:read", + tenant: "tenant-alpha"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "advisory:ingest"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}"); + var grantedScopes = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]); + Assert.Equal(new[] { "advisory:ingest" }, grantedScopes); + } + + [Fact] + public async Task ValidateClientCredentials_RejectsEffectiveWrite_WhenServiceIdentityMissing() + { + var clientDocument = CreateClient( + clientId: "policy-engine", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "effective:write findings:read policy:run", + tenant: "tenant-default"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + Assert.True(clientDocument.Properties.ContainsKey(AuthorityClientMetadataKeys.Tenant)); + Assert.Equal("tenant-default", clientDocument.Properties[AuthorityClientMetadataKeys.Tenant]); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "effective:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.UnauthorizedClient, context.Error); + Assert.Equal("Scope 'effective:write' is reserved for the Policy Engine service identity.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_RejectsEffectiveWrite_WhenTenantMissing() + { + var clientDocument = CreateClient( + clientId: "policy-engine", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "effective:write findings:read policy:run"); + clientDocument.Properties[AuthorityClientMetadataKeys.ServiceIdentity] = StellaOpsServiceIdentities.PolicyEngine; + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "effective:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidClient, context.Error); + Assert.Equal("Policy Engine service identity requires a tenant assignment.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_AllowsEffectiveWrite_ForPolicyEngineServiceIdentity() + { + var clientDocument = CreateClient( + clientId: "policy-engine", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "effective:write findings:read policy:run", + tenant: "tenant-default"); + clientDocument.Properties[AuthorityClientMetadataKeys.ServiceIdentity] = StellaOpsServiceIdentities.PolicyEngine; + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "effective:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}"); + var grantedScopes = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]); + Assert.Equal(new[] { "effective:write" }, grantedScopes); + + var tenant = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty]); + Assert.Equal("tenant-default", tenant); + } + + [Fact] + public async Task ValidateClientCredentials_RejectsGraphWrite_WhenServiceIdentityMissing() + { + var clientDocument = CreateClient( + clientId: "cartographer-service", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "graph:write graph:read", + tenant: "tenant-default"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "graph:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.UnauthorizedClient, context.Error); + Assert.Equal("Scope 'graph:write' is reserved for the Cartographer service identity.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_RejectsGraphWrite_WhenServiceIdentityMismatch() + { + var clientDocument = CreateClient( + clientId: "cartographer-service", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "graph:write graph:read", + tenant: "tenant-default"); + clientDocument.Properties[AuthorityClientMetadataKeys.ServiceIdentity] = StellaOpsServiceIdentities.PolicyEngine; + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "graph:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.UnauthorizedClient, context.Error); + Assert.Equal("Scope 'graph:write' is reserved for the Cartographer service identity.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_RejectsGraphScopes_WhenTenantMissing() + { + var clientDocument = CreateClient( + clientId: "graph-api", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "graph:read graph:export"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "graph:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidClient, context.Error); + Assert.Equal("Graph scopes require a tenant assignment.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_AllowsGraphRead_WithTenant() + { + var clientDocument = CreateClient( + clientId: "graph-api", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "graph:read graph:export", + tenant: "tenant-default"); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "graph:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}"); + var grantedScopes = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]); + Assert.Equal(new[] { "graph:read" }, grantedScopes); + var tenant = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty]); + Assert.Equal("tenant-default", tenant); + } + + [Fact] + public async Task ValidateClientCredentials_AllowsGraphWrite_ForCartographerServiceIdentity() + { + var clientDocument = CreateClient( + clientId: "cartographer-service", + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "graph:write graph:read", + tenant: "tenant-default"); + clientDocument.Properties[AuthorityClientMetadataKeys.ServiceIdentity] = StellaOpsServiceIdentities.Cartographer; + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "graph:write"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}"); + var grantedScopes = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]); + Assert.Equal(new[] { "graph:write" }, grantedScopes); + var tenant = Assert.IsType(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty]); + Assert.Equal("tenant-default", tenant); + } + [Fact] public async Task ValidateClientCredentials_EmitsTamperAuditEvent_WhenUnexpectedParametersPresent() { @@ -231,6 +549,7 @@ public class ClientCredentialsHandlersTests registry, tokenStore, sessionAccessor, + rateMetadata, TimeProvider.System, TestActivitySource, NullLogger.Instance); @@ -550,7 +869,8 @@ public class ClientCredentialsHandlersTests clientType: "public", allowedGrantTypes: "client_credentials", allowedScopes: "jobs:trigger", - allowedAudiences: "signer"); + allowedAudiences: "signer", + tenant: "Tenant-Alpha"); var descriptor = CreateDescriptor(clientDocument); var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: descriptor); @@ -582,6 +902,7 @@ public class ClientCredentialsHandlersTests registry, tokenStore, sessionAccessor, + metadataAccessor, TimeProvider.System, TestActivitySource, NullLogger.Instance); @@ -601,6 +922,7 @@ public class ClientCredentialsHandlersTests Assert.Equal(clientDocument.Plugin, identityProviderClaim); var principal = context.Principal ?? throw new InvalidOperationException("Principal missing"); + Assert.Equal("tenant-alpha", principal.FindFirstValue(StellaOpsClaimTypes.Tenant)); var tokenId = principal.GetClaim(OpenIddictConstants.Claims.JwtId); Assert.False(string.IsNullOrWhiteSpace(tokenId)); @@ -616,6 +938,7 @@ public class ClientCredentialsHandlersTests Assert.Equal(tokenId, persisted.TokenId); Assert.Equal(clientDocument.ClientId, persisted.ClientId); Assert.Equal("valid", persisted.Status); + Assert.Equal("tenant-alpha", persisted.Tenant); Assert.Equal(new[] { "jobs:trigger" }, persisted.Scope); } } @@ -1169,6 +1492,12 @@ internal sealed class TestRateLimiterMetadataAccessor : IAuthorityRateLimiterMet public void SetSubjectId(string? subjectId) => metadata.SubjectId = subjectId; + public void SetTenant(string? tenant) + { + metadata.Tenant = string.IsNullOrWhiteSpace(tenant) ? null : tenant.Trim().ToLowerInvariant(); + metadata.SetTag("authority.tenant", metadata.Tenant); + } + public void SetTag(string name, string? value) => metadata.SetTag(name, value); } @@ -1238,15 +1567,17 @@ internal static class TestHelpers } public static AuthorityClientDocument CreateClient( + string clientId = "concelier", string? secret = "s3cr3t!", string clientType = "confidential", string allowedGrantTypes = "client_credentials", string allowedScopes = "jobs:read", - string allowedAudiences = "") + string allowedAudiences = "", + string? tenant = null) { var document = new AuthorityClientDocument { - ClientId = "concelier", + ClientId = clientId, ClientType = clientType, SecretHash = secret is null ? null : AuthoritySecretHasher.ComputeHash(secret), Plugin = "standard", @@ -1262,9 +1593,18 @@ internal static class TestHelpers document.Properties[AuthorityClientMetadataKeys.Audiences] = allowedAudiences; } + var normalizedTenant = NormalizeTenant(tenant); + if (normalizedTenant is not null) + { + document.Properties[AuthorityClientMetadataKeys.Tenant] = normalizedTenant; + } + return document; } + private static string? NormalizeTenant(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); + public static AuthorityClientDescriptor CreateDescriptor(AuthorityClientDocument document) { var allowedGrantTypes = document.Properties.TryGetValue(AuthorityClientMetadataKeys.AllowedGrantTypes, out var grants) ? grants?.Split(' ', StringSplitOptions.RemoveEmptyEntries) : Array.Empty(); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs index 28d3e8ef..7ee0d461 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/PasswordGrantHandlersTests.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.DependencyInjection; +using MongoDB.Driver; using OpenIddict.Abstractions; using OpenIddict.Server; using OpenIddict.Server.AspNetCore; @@ -15,6 +16,8 @@ using StellaOps.Authority.OpenIddict; using StellaOps.Authority.OpenIddict.Handlers; using StellaOps.Authority.Plugins.Abstractions; using StellaOps.Authority.RateLimiting; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; using StellaOps.Cryptography.Audit; using Xunit; @@ -30,15 +33,20 @@ public class PasswordGrantHandlersTests var sink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); var registry = CreateRegistry(new SuccessCredentialStore()); - var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); - var handle = new HandlePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); + var clientStore = new StubClientStore(CreateClientDocument()); + var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, clientStore, TimeProvider.System, NullLogger.Instance); + var handle = new HandlePasswordGrantHandler(registry, clientStore, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); var transaction = CreatePasswordTransaction("alice", "Password1!"); await validate.HandleAsync(new OpenIddictServerEvents.ValidateTokenRequestContext(transaction)); await handle.HandleAsync(new OpenIddictServerEvents.HandleTokenRequestContext(transaction)); - Assert.Contains(sink.Events, record => record.EventType == "authority.password.grant" && record.Outcome == AuthEventOutcome.Success); + var successEvent = Assert.Single(sink.Events, record => record.EventType == "authority.password.grant" && record.Outcome == AuthEventOutcome.Success); + Assert.Equal("tenant-alpha", successEvent.Tenant.Value); + + var metadata = metadataAccessor.GetMetadata(); + Assert.Equal("tenant-alpha", metadata?.Tenant); } [Fact] @@ -47,8 +55,9 @@ public class PasswordGrantHandlersTests var sink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); var registry = CreateRegistry(new FailureCredentialStore()); - var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); - var handle = new HandlePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); + var clientStore = new StubClientStore(CreateClientDocument()); + var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, clientStore, TimeProvider.System, NullLogger.Instance); + var handle = new HandlePasswordGrantHandler(registry, clientStore, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); var transaction = CreatePasswordTransaction("alice", "BadPassword!"); @@ -64,8 +73,9 @@ public class PasswordGrantHandlersTests var sink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); var registry = CreateRegistry(new LockoutCredentialStore()); - var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); - var handle = new HandlePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); + var clientStore = new StubClientStore(CreateClientDocument()); + var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, clientStore, TimeProvider.System, NullLogger.Instance); + var handle = new HandlePasswordGrantHandler(registry, clientStore, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); var transaction = CreatePasswordTransaction("alice", "Locked!"); @@ -81,7 +91,8 @@ public class PasswordGrantHandlersTests var sink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); var registry = CreateRegistry(new SuccessCredentialStore()); - var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger.Instance); + var clientStore = new StubClientStore(CreateClientDocument()); + var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, clientStore, TimeProvider.System, NullLogger.Instance); var transaction = CreatePasswordTransaction("alice", "Password1!"); transaction.Request?.SetParameter("unexpected_param", "value"); @@ -113,7 +124,9 @@ public class PasswordGrantHandlersTests { GrantType = OpenIddictConstants.GrantTypes.Password, Username = username, - Password = password + Password = password, + ClientId = "cli-app", + Scope = "jobs:trigger" }; return new OpenIddictServerTransaction @@ -124,6 +137,21 @@ public class PasswordGrantHandlersTests }; } + private static AuthorityClientDocument CreateClientDocument() + { + var document = new AuthorityClientDocument + { + ClientId = "cli-app", + ClientType = "public" + }; + + document.Properties[AuthorityClientMetadataKeys.AllowedGrantTypes] = "password"; + document.Properties[AuthorityClientMetadataKeys.AllowedScopes] = "jobs:trigger"; + document.Properties[AuthorityClientMetadataKeys.Tenant] = "tenant-alpha"; + + return document; + } + private sealed class StubIdentityProviderPlugin : IIdentityProviderPlugin { public StubIdentityProviderPlugin(string name, IUserCredentialStore store) @@ -220,4 +248,26 @@ public class PasswordGrantHandlersTests => ValueTask.FromResult(null); } + private sealed class StubClientStore : IAuthorityClientStore + { + private readonly Dictionary clients; + + public StubClientStore(params AuthorityClientDocument[] documents) + { + clients = documents.ToDictionary(static doc => doc.ClientId, doc => doc, StringComparer.OrdinalIgnoreCase); + } + + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + clients.TryGetValue(clientId, out var document); + return ValueTask.FromResult(document); + } + + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => throw new NotImplementedException(); + + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => throw new NotImplementedException(); + } + } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs index 2141b8d3..a914bcf1 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs @@ -52,7 +52,8 @@ public sealed class TokenPersistenceIntegrationTests var clientDocument = TestHelpers.CreateClient( secret: "s3cr3t!", allowedGrantTypes: "client_credentials", - allowedScopes: "jobs:trigger jobs:read"); + allowedScopes: "jobs:trigger jobs:read", + tenant: "tenant-alpha"); await clientStore.UpsertAsync(clientDocument, CancellationToken.None); @@ -66,7 +67,7 @@ public sealed class TokenPersistenceIntegrationTests var sessionAccessor = scope.ServiceProvider.GetRequiredService(); var options = TestHelpers.CreateAuthorityOptions(); var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, new NoopCertificateValidator(), new HttpContextAccessor(), options, NullLogger.Instance); - var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, sessionAccessor, clock, TestActivitySource, NullLogger.Instance); + var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, sessionAccessor, metadataAccessor, clock, TestActivitySource, NullLogger.Instance); var persistHandler = new PersistTokensHandler(tokenStore, sessionAccessor, clock, TestActivitySource, NullLogger.Instance); var transaction = TestHelpers.CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:trigger"); @@ -83,6 +84,7 @@ public sealed class TokenPersistenceIntegrationTests var principal = Assert.IsType(handleContext.Principal); var tokenId = principal.GetClaim(OpenIddictConstants.Claims.JwtId); Assert.False(string.IsNullOrWhiteSpace(tokenId)); + Assert.Equal("tenant-alpha", metadataAccessor.GetMetadata()?.Tenant); var signInContext = new OpenIddictServerEvents.ProcessSignInContext(transaction) { @@ -100,6 +102,7 @@ public sealed class TokenPersistenceIntegrationTests Assert.Equal(issuedAt, stored.CreatedAt); Assert.Equal(issuedAt.AddMinutes(15), stored.ExpiresAt); Assert.Equal(new[] { "jobs:trigger" }, stored.Scope); + Assert.Equal("tenant-alpha", stored.Tenant); } [Fact] @@ -119,7 +122,8 @@ public sealed class TokenPersistenceIntegrationTests secret: null, clientType: "public", allowedGrantTypes: "password refresh_token", - allowedScopes: "openid profile jobs:read"); + allowedScopes: "openid profile jobs:read", + tenant: "tenant-alpha"); await clientStore.UpsertAsync(clientDocument, CancellationToken.None); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Permalinks/VulnPermalinkServiceTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Permalinks/VulnPermalinkServiceTests.cs new file mode 100644 index 00000000..7e6429ae --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Permalinks/VulnPermalinkServiceTests.cs @@ -0,0 +1,150 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Authority.Permalinks; +using StellaOps.Authority.Signing; +using StellaOps.Configuration; +using StellaOps.Cryptography; +using StellaOps.Cryptography.DependencyInjection; +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Authority.Tests.Permalinks; + +public sealed class VulnPermalinkServiceTests +{ + [Fact] + public async Task CreateAsync_IssuesSignedTokenWithExpectedClaims() + { + var tempDir = Directory.CreateTempSubdirectory("authority-permalink-tests").FullName; + var keyRelative = "permalink.pem"; + try + { + CreateEcPrivateKey(Path.Combine(tempDir, keyRelative)); + + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test"), + Storage = { ConnectionString = "mongodb://localhost/test" }, + Signing = + { + Enabled = true, + ActiveKeyId = "permalink-key", + KeyPath = keyRelative, + Algorithm = SignatureAlgorithms.Es256, + KeySource = "file", + Provider = "default" + } + }; + + var fakeTime = new FakeTimeProvider(DateTimeOffset.Parse("2025-10-26T12:00:00Z")); + + using var provider = BuildProvider(tempDir, options, fakeTime); + // Ensure signing keys are loaded + provider.GetRequiredService(); + + var service = provider.GetRequiredService(); + var state = JsonDocument.Parse("{\"vulnId\":\"CVE-2025-1234\"}").RootElement; + var request = new VulnPermalinkRequest( + Tenant: "tenant-a", + ResourceKind: "vulnerability", + State: state, + ExpiresInSeconds: null, + Environment: "prod"); + + var expectedNow = fakeTime.GetUtcNow(); + + var response = await service.CreateAsync(request, default); + + Assert.NotNull(response.Token); + Assert.Equal(expectedNow, response.IssuedAt); + Assert.Equal(expectedNow.AddHours(24), response.ExpiresAt); + Assert.Contains(StellaOpsScopes.VulnRead, response.Scopes); + + var parts = response.Token.Split('.'); + Assert.Equal(3, parts.Length); + + var payloadBytes = Base64UrlEncoder.DecodeBytes(parts[1]); + using var payloadDocument = JsonDocument.Parse(payloadBytes); + var payload = payloadDocument.RootElement; + + Assert.Equal("vulnerability", payload.GetProperty("type").GetString()); + Assert.Equal("tenant-a", payload.GetProperty("tenant").GetString()); + Assert.Equal("prod", payload.GetProperty("environment").GetString()); + Assert.Equal(expectedNow.ToUnixTimeSeconds(), payload.GetProperty("iat").GetInt64()); + Assert.Equal(expectedNow.ToUnixTimeSeconds(), payload.GetProperty("nbf").GetInt64()); + Assert.Equal(expectedNow.AddHours(24).ToUnixTimeSeconds(), payload.GetProperty("exp").GetInt64()); + + var scopes = payload.GetProperty("scopes").EnumerateArray().Select(element => element.GetString()).ToArray(); + Assert.Contains(StellaOpsScopes.VulnRead, scopes); + + var resource = payload.GetProperty("resource"); + Assert.Equal("vulnerability", resource.GetProperty("kind").GetString()); + Assert.Equal("CVE-2025-1234", resource.GetProperty("state").GetProperty("vulnId").GetString()); + } + finally + { + try + { + Directory.Delete(tempDir, recursive: true); + } + catch + { + // ignore cleanup failures + } + } + } + + private static ServiceProvider BuildProvider(string basePath, StellaOpsAuthorityOptions options, TimeProvider timeProvider) + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.SetMinimumLevel(LogLevel.Debug)); + services.AddSingleton(new TestHostEnvironment(basePath)); + services.AddSingleton(options); + services.AddSingleton>(Options.Create(options)); + services.AddSingleton(timeProvider); + services.AddStellaOpsCrypto(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.AddSingleton(); + services.AddSingleton(); + + return services.BuildServiceProvider(); + } + + private static void CreateEcPrivateKey(string path) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + using var ecdsa = System.Security.Cryptography.ECDsa.Create(System.Security.Cryptography.ECCurve.NamedCurves.nistP256); + var pem = ecdsa.ExportECPrivateKeyPem(); + File.WriteAllText(path, pem); + } + + private sealed class TestHostEnvironment : IHostEnvironment + { + public TestHostEnvironment(string contentRoot) + { + ContentRootPath = contentRoot; + ContentRootFileProvider = new PhysicalFileProvider(contentRoot); + EnvironmentName = Environments.Development; + ApplicationName = "StellaOps.Authority.Tests"; + } + + public string EnvironmentName { get; set; } + + public string ApplicationName { get; set; } + + public string ContentRootPath { get; set; } + + public IFileProvider ContentRootFileProvider { get; set; } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs index be40594c..01e6b10e 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterMetadataAccessorTests.cs @@ -18,6 +18,7 @@ public class AuthorityRateLimiterMetadataAccessorTests accessor.SetClientId("client-123"); accessor.SetTag("custom", "tag"); accessor.SetSubjectId("subject-1"); + accessor.SetTenant("Tenant-Alpha"); var metadata = accessor.GetMetadata(); Assert.NotNull(metadata); @@ -25,6 +26,8 @@ public class AuthorityRateLimiterMetadataAccessorTests Assert.Equal("subject-1", metadata.SubjectId); Assert.Equal("client-123", metadata.Tags["authority.client_id"]); Assert.Equal("subject-1", metadata.Tags["authority.subject_id"]); + Assert.Equal("tenant-alpha", metadata.Tenant); + Assert.Equal("tenant-alpha", metadata.Tags["authority.tenant"]); Assert.Equal("tag", metadata.Tags["custom"]); } } diff --git a/src/StellaOps.Authority/StellaOps.Authority/Audit/AuthorityAuditSink.cs b/src/StellaOps.Authority/StellaOps.Authority/Audit/AuthorityAuditSink.cs index 43fcbc8c..3b7451f9 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Audit/AuthorityAuditSink.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Audit/AuthorityAuditSink.cs @@ -60,6 +60,11 @@ internal sealed class AuthorityAuditSink : IAuthEventSink OccurredAt = record.OccurredAt }; + if (record.Tenant.HasValue) + { + document.Tenant = record.Tenant.Value; + } + if (record.Scopes is { Count: > 0 }) { document.Scopes = record.Scopes @@ -142,6 +147,8 @@ internal sealed class AuthorityAuditSink : IAuthEventSink AddClassified(entries, "audit.client.provider", client.Provider); } + AddClassified(entries, "audit.tenant", record.Tenant); + if (record.Network is { } network) { AddClassified(entries, "audit.network.remote", network.RemoteAddress); diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs index 883ede91..7abbec85 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs @@ -14,15 +14,16 @@ internal static class AuthorityOpenIddictConstants internal const string AuditConfidentialProperty = "authority:audit_confidential"; internal const string AuditRequestedScopesProperty = "authority:audit_requested_scopes"; internal const string AuditGrantedScopesProperty = "authority:audit_granted_scopes"; - internal const string AuditInvalidScopeProperty = "authority:audit_invalid_scope"; - internal const string ClientSenderConstraintProperty = "authority:client_sender_constraint"; - internal const string SenderConstraintProperty = "authority:sender_constraint"; - internal const string DpopKeyThumbprintProperty = "authority:dpop_thumbprint"; - internal const string DpopProofJwtIdProperty = "authority:dpop_jti"; - internal const string DpopIssuedAtProperty = "authority:dpop_iat"; - internal const string DpopConsumedNonceProperty = "authority:dpop_nonce"; - internal const string ConfirmationClaimType = "cnf"; - internal const string SenderConstraintClaimType = "authority_sender_constraint"; - internal const string MtlsCertificateThumbprintProperty = "authority:mtls_thumbprint"; - internal const string MtlsCertificateHexProperty = "authority:mtls_thumbprint_hex"; -} + internal const string AuditInvalidScopeProperty = "authority:audit_invalid_scope"; + internal const string ClientSenderConstraintProperty = "authority:client_sender_constraint"; + internal const string SenderConstraintProperty = "authority:sender_constraint"; + internal const string DpopKeyThumbprintProperty = "authority:dpop_thumbprint"; + internal const string DpopProofJwtIdProperty = "authority:dpop_jti"; + internal const string DpopIssuedAtProperty = "authority:dpop_iat"; + internal const string DpopConsumedNonceProperty = "authority:dpop_nonce"; + internal const string ConfirmationClaimType = "cnf"; + internal const string SenderConstraintClaimType = "authority_sender_constraint"; + internal const string MtlsCertificateThumbprintProperty = "authority:mtls_thumbprint"; + internal const string MtlsCertificateHexProperty = "authority:mtls_thumbprint_hex"; + internal const string ClientTenantProperty = "authority:client_tenant"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsAuditHelper.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsAuditHelper.cs index e2e01f70..03592dda 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsAuditHelper.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsAuditHelper.cs @@ -39,6 +39,7 @@ internal static class ClientCredentialsAuditHelper string? reason, string? clientId, string? providerName, + string? tenant, bool? confidential, IReadOnlyList requestedScopes, IReadOnlyList grantedScopes, @@ -54,6 +55,7 @@ internal static class ClientCredentialsAuditHelper var network = BuildNetwork(metadata); var normalizedGranted = NormalizeScopes(grantedScopes); var properties = BuildProperties(confidential, requestedScopes, invalidScope, extraProperties); + var normalizedTenant = NormalizeTenant(tenant); return new AuthEventRecord { @@ -66,6 +68,7 @@ internal static class ClientCredentialsAuditHelper Client = client, Scopes = normalizedGranted, Network = network, + Tenant = ClassifiedString.Public(normalizedTenant), Properties = properties }; } @@ -76,6 +79,7 @@ internal static class ClientCredentialsAuditHelper AuthorityRateLimiterMetadata? metadata, string? clientId, string? providerName, + string? tenant, bool? confidential, IEnumerable unexpectedParameters) { @@ -127,6 +131,7 @@ internal static class ClientCredentialsAuditHelper reason: reason, clientId: clientId, providerName: providerName, + tenant: tenant, confidential: confidential, requestedScopes: Array.Empty(), grantedScopes: Array.Empty(), @@ -249,4 +254,7 @@ internal static class ClientCredentialsAuditHelper private static string? Normalize(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string? NormalizeTenant(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); } diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs index b8037cdc..79bc75b6 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs @@ -95,14 +95,15 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle if (unexpectedParameters.Count > 0) { var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString(); - var tamperRecord = ClientCredentialsAuditHelper.CreateTamperRecord( - timeProvider, - context.Transaction, - metadata, - clientId, - providerHint, - confidential: null, - unexpectedParameters); + var tamperRecord = ClientCredentialsAuditHelper.CreateTamperRecord( + timeProvider, + context.Transaction, + metadata, + clientId, + providerHint, + tenant: metadata?.Tenant, + confidential: null, + unexpectedParameters); await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false); } @@ -250,20 +251,111 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle return; } - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditGrantedScopesProperty] = resolvedScopes.Scopes; - - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; + var grantedScopes = resolvedScopes.Scopes; + + bool EnsureTenantAssigned() + { + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTenantProperty, out var tenantObj) && + tenantObj is string existingTenant && + !string.IsNullOrWhiteSpace(existingTenant)) + { + return true; + } + + if (document.Properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var tenantProperty)) + { + var normalizedTenant = ClientCredentialHandlerHelpers.NormalizeTenant(tenantProperty); + if (normalizedTenant is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty] = normalizedTenant; + metadataAccessor.SetTenant(normalizedTenant); + activity?.SetTag("authority.tenant", normalizedTenant); + return true; + } + } + + context.Transaction.Properties.Remove(AuthorityOpenIddictConstants.ClientTenantProperty); + return false; + } + + var hasGraphRead = grantedScopes.Length > 0 && Array.IndexOf(grantedScopes, StellaOpsScopes.GraphRead) >= 0; + var hasGraphWrite = grantedScopes.Length > 0 && Array.IndexOf(grantedScopes, StellaOpsScopes.GraphWrite) >= 0; + var hasGraphExport = grantedScopes.Length > 0 && Array.IndexOf(grantedScopes, StellaOpsScopes.GraphExport) >= 0; + var hasGraphSimulate = grantedScopes.Length > 0 && Array.IndexOf(grantedScopes, StellaOpsScopes.GraphSimulate) >= 0; + var graphScopesRequested = hasGraphRead || hasGraphWrite || hasGraphExport || hasGraphSimulate; + + var tenantScopeForAudit = hasGraphWrite + ? StellaOpsScopes.GraphWrite + : hasGraphExport + ? StellaOpsScopes.GraphExport + : hasGraphSimulate + ? StellaOpsScopes.GraphSimulate + : StellaOpsScopes.GraphRead; + + if (graphScopesRequested && !EnsureTenantAssigned()) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditInvalidScopeProperty] = tenantScopeForAudit; + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Graph scopes require a tenant assignment."); + logger.LogWarning( + "Client credentials validation failed for {ClientId}: graph scopes require tenant assignment.", + document.ClientId); + return; + } + + if (grantedScopes.Length > 0 && + Array.IndexOf(grantedScopes, StellaOpsScopes.EffectiveWrite) >= 0) + { + if (!document.Properties.TryGetValue(AuthorityClientMetadataKeys.ServiceIdentity, out var serviceIdentity) || + string.IsNullOrWhiteSpace(serviceIdentity) || + !string.Equals(serviceIdentity.Trim(), StellaOpsServiceIdentities.PolicyEngine, StringComparison.OrdinalIgnoreCase)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditInvalidScopeProperty] = StellaOpsScopes.EffectiveWrite; + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Scope 'effective:write' is reserved for the Policy Engine service identity."); + logger.LogWarning( + "Client credentials validation failed for {ClientId}: effective:write scope requires Policy Engine service identity marker.", + document.ClientId); + return; + } + + if (!EnsureTenantAssigned()) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Policy Engine service identity requires a tenant assignment."); + logger.LogWarning( + "Client credentials validation failed for {ClientId}: effective:write scope requires tenant assignment.", + document.ClientId); + return; + } + } + + if (hasGraphWrite) + { + if (!document.Properties.TryGetValue(AuthorityClientMetadataKeys.ServiceIdentity, out var serviceIdentity) || + string.IsNullOrWhiteSpace(serviceIdentity) || + !string.Equals(serviceIdentity.Trim(), StellaOpsServiceIdentities.Cartographer, StringComparison.OrdinalIgnoreCase)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditInvalidScopeProperty] = StellaOpsScopes.GraphWrite; + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Scope 'graph:write' is reserved for the Cartographer service identity."); + logger.LogWarning( + "Client credentials validation failed for {ClientId}: graph:write scope requires Cartographer service identity marker.", + document.ClientId); + return; + } + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditGrantedScopesProperty] = grantedScopes; + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; if (providerMetadata is not null) { context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = providerMetadata.Name; activity?.SetTag("authority.identity_provider", providerMetadata.Name); } - - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; - logger.LogInformation("Client credentials validated for {ClientId}.", document.ClientId); - } - finally - { + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; + logger.LogInformation("Client credentials validated for {ClientId}.", document.ClientId); + } + finally + { var outcome = context.IsRejected ? AuthEventOutcome.Failure : AuthEventOutcome.Success; var reason = context.IsRejected ? context.ErrorDescription : null; var auditClientId = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditClientIdProperty, out var clientValue) @@ -281,23 +373,27 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle var granted = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditGrantedScopesProperty, out var grantedValue) && grantedValue is string[] grantedArray ? (IReadOnlyList)grantedArray : Array.Empty(); - var invalidScope = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditInvalidScopeProperty, out var invalidValue) - ? invalidValue as string - : null; - - var record = ClientCredentialsAuditHelper.CreateRecord( - timeProvider, - context.Transaction, - metadata, + var invalidScope = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditInvalidScopeProperty, out var invalidValue) + ? invalidValue as string + : null; + var tenantValueForAudit = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTenantProperty, out var tenantAuditObj) && tenantAuditObj is string tenantAudit + ? tenantAudit + : metadata?.Tenant; + + var record = ClientCredentialsAuditHelper.CreateRecord( + timeProvider, + context.Transaction, + metadata, null, - outcome, - reason, - auditClientId, - providerName, - confidentialValue, - requested, - granted, - invalidScope); + outcome, + reason, + auditClientId, + providerName, + tenantValueForAudit, + confidentialValue, + requested, + granted, + invalidScope); await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); } @@ -390,31 +486,34 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle return set; } } - -internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler -{ - private readonly IAuthorityIdentityProviderRegistry registry; - private readonly IAuthorityTokenStore tokenStore; - private readonly IAuthorityMongoSessionAccessor sessionAccessor; - private readonly TimeProvider clock; - private readonly ActivitySource activitySource; - private readonly ILogger logger; - - public HandleClientCredentialsHandler( - IAuthorityIdentityProviderRegistry registry, - IAuthorityTokenStore tokenStore, - IAuthorityMongoSessionAccessor sessionAccessor, - TimeProvider clock, - ActivitySource activitySource, - ILogger logger) - { - this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); - this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); - this.sessionAccessor = sessionAccessor ?? throw new ArgumentNullException(nameof(sessionAccessor)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } + +internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly IAuthorityTokenStore tokenStore; + private readonly IAuthorityMongoSessionAccessor sessionAccessor; + private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; + private readonly TimeProvider clock; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public HandleClientCredentialsHandler( + IAuthorityIdentityProviderRegistry registry, + IAuthorityTokenStore tokenStore, + IAuthorityMongoSessionAccessor sessionAccessor, + IAuthorityRateLimiterMetadataAccessor metadataAccessor, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.sessionAccessor = sessionAccessor ?? throw new ArgumentNullException(nameof(sessionAccessor)); + this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } public async ValueTask HandleAsync(OpenIddictServerEvents.HandleTokenRequestContext context) { @@ -468,21 +567,41 @@ internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler< activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); var tokenId = identity.GetClaim(OpenIddictConstants.Claims.JwtId); - if (string.IsNullOrEmpty(tokenId)) - { - tokenId = Guid.NewGuid().ToString("N"); - identity.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); - } - - identity.SetDestinations(static claim => claim.Type switch - { - OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken }, - OpenIddictConstants.Claims.ClientId => new[] { OpenIddictConstants.Destinations.AccessToken }, - OpenIddictConstants.Claims.JwtId => new[] { OpenIddictConstants.Destinations.AccessToken }, - StellaOpsClaimTypes.IdentityProvider => new[] { OpenIddictConstants.Destinations.AccessToken }, - _ => new[] { OpenIddictConstants.Destinations.AccessToken } - }); - + if (string.IsNullOrEmpty(tokenId)) + { + tokenId = Guid.NewGuid().ToString("N"); + identity.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); + } + + identity.SetDestinations(static claim => claim.Type switch + { + OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.ClientId => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.JwtId => new[] { OpenIddictConstants.Destinations.AccessToken }, + StellaOpsClaimTypes.IdentityProvider => new[] { OpenIddictConstants.Destinations.AccessToken }, + _ => new[] { OpenIddictConstants.Destinations.AccessToken } + }); + + string? tenant = null; + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTenantProperty, out var tenantValue) && + tenantValue is string storedTenant && + !string.IsNullOrWhiteSpace(storedTenant)) + { + tenant = storedTenant; + } + else if (document.Properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var tenantProperty)) + { + tenant = ClientCredentialHandlerHelpers.NormalizeTenant(tenantProperty); + } + + if (!string.IsNullOrWhiteSpace(tenant)) + { + identity.SetClaim(StellaOpsClaimTypes.Tenant, tenant); + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty] = tenant; + metadataAccessor.SetTenant(tenant); + activity?.SetTag("authority.tenant", tenant); + } + var (providerHandle, descriptor) = await ResolveProviderAsync(context, document).ConfigureAwait(false); if (context.IsRejected) { @@ -541,6 +660,14 @@ internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler< { var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user: null, descriptor); await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(descriptor.Tenant)) + { + identity.SetClaim(StellaOpsClaimTypes.Tenant, descriptor.Tenant); + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty] = descriptor.Tenant; + metadataAccessor.SetTenant(descriptor.Tenant); + activity?.SetTag("authority.tenant", descriptor.Tenant); + } } var session = await sessionAccessor.GetSessionAsync(context.CancellationToken).ConfigureAwait(false); @@ -667,15 +794,22 @@ internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler< senderThumbprint = mtlsThumbprint; } - if (senderThumbprint is not null) - { - record.SenderKeyThumbprint = senderThumbprint; - } - - if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.DpopConsumedNonceProperty, out var nonceObj) && - nonceObj is string nonce && - !string.IsNullOrWhiteSpace(nonce)) - { + if (senderThumbprint is not null) + { + record.SenderKeyThumbprint = senderThumbprint; + } + + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTenantProperty, out var tenantObj) && + tenantObj is string tenantValue && + !string.IsNullOrWhiteSpace(tenantValue)) + { + record.Tenant = tenantValue; + } + + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.DpopConsumedNonceProperty, out var nonceObj) && + nonceObj is string nonce && + !string.IsNullOrWhiteSpace(nonce)) + { record.SenderNonce = nonce; } @@ -739,15 +873,18 @@ internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler< internal static class ClientCredentialHandlerHelpers { - public static IReadOnlyList Split(IReadOnlyDictionary properties, string key) - { - if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) - { - return Array.Empty(); - } - - return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - } + public static IReadOnlyList Split(IReadOnlyDictionary properties, string key) + { + if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + public static string? NormalizeTenant(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); public static (string[] Scopes, string? InvalidScope) ResolveGrantedScopes( IReadOnlyCollection allowedScopes, diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs index 52179e0e..ba109812 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs @@ -624,31 +624,35 @@ internal sealed class ValidateDpopProofHandler : IOpenIddictServerHandler(), - grantedScopes: Array.Empty(), - invalidScope: null, - extraProperties: properties, + if (nonceExpiresAt is { } expiresAt) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.nonce.expires_at", + Value = ClassifiedString.Public(expiresAt.ToString("O", CultureInfo.InvariantCulture)) + }); + } + + var confidential = string.Equals(clientDocument.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); + var tenant = clientDocument.Properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var tenantValue) + ? tenantValue?.Trim().ToLowerInvariant() + : null; + + var record = ClientCredentialsAuditHelper.CreateRecord( + clock, + context.Transaction, + metadata, + clientSecret: null, + outcome, + reason, + clientDocument.ClientId, + providerName: clientDocument.Plugin, + tenant, + confidential, + requestedScopes: Array.Empty(), + grantedScopes: Array.Empty(), + invalidScope: null, + extraProperties: properties, eventType: eventType); await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs index 9f652a61..92739be1 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/PasswordGrantHandlers.cs @@ -10,9 +10,12 @@ using OpenIddict.Abstractions; using OpenIddict.Extensions; using OpenIddict.Server; using OpenIddict.Server.AspNetCore; +using StellaOps.Auth.Abstractions; using StellaOps.Authority.OpenIddict; using StellaOps.Authority.Plugins.Abstractions; using StellaOps.Authority.RateLimiting; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; using StellaOps.Cryptography.Audit; namespace StellaOps.Authority.OpenIddict.Handlers; @@ -23,6 +26,7 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler logger; @@ -31,6 +35,7 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler logger) { @@ -38,6 +43,7 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler() : requestedScopesInput.ToArray(); + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditRequestedScopesProperty] = requestedScopes; + + if (string.IsNullOrWhiteSpace(clientId)) + { + var record = PasswordGrantAuditHelper.CreatePasswordGrantRecord( + timeProvider, + context.Transaction, + metadata, + AuthEventOutcome.Failure, + "Client identifier is required for password grant.", + clientId: null, + providerName: null, + tenant: null, + user: null, + username: context.Request.Username, + scopes: requestedScopes, + retryAfter: null, + failureCode: AuthorityCredentialFailureCode.InvalidCredentials, + extraProperties: null); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client identifier is required."); + logger.LogWarning("Password grant validation failed: missing client_id for {Username}.", context.Request.Username); + return; + } + + var clientDocument = await clientStore.FindByClientIdAsync(clientId, context.CancellationToken).ConfigureAwait(false); + if (clientDocument is null || clientDocument.Disabled) + { + var record = PasswordGrantAuditHelper.CreatePasswordGrantRecord( + timeProvider, + context.Transaction, + metadata, + AuthEventOutcome.Failure, + "Client is not permitted for password grant.", + clientId, + providerName: null, + tenant: null, + user: null, + username: context.Request.Username, + scopes: requestedScopes, + retryAfter: null, + failureCode: AuthorityCredentialFailureCode.InvalidCredentials, + extraProperties: null); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + + context.Reject(OpenIddictConstants.Errors.InvalidClient, "The specified client is not permitted."); + logger.LogWarning("Password grant validation failed: client {ClientId} disabled or missing.", clientId); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = clientDocument; + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditClientIdProperty] = clientId; + + var tenant = PasswordGrantAuditHelper.NormalizeTenant(clientDocument.Properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var tenantValue) ? tenantValue : null); + if (!string.IsNullOrWhiteSpace(tenant)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty] = tenant; + metadataAccessor.SetTenant(tenant); + activity?.SetTag("authority.tenant", tenant); + } + + var allowedGrantTypes = ClientCredentialHandlerHelpers.Split(clientDocument.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); + if (allowedGrantTypes.Count > 0 && + !allowedGrantTypes.Any(static grant => string.Equals(grant, OpenIddictConstants.GrantTypes.Password, StringComparison.Ordinal))) + { + var record = PasswordGrantAuditHelper.CreatePasswordGrantRecord( + timeProvider, + context.Transaction, + metadata, + AuthEventOutcome.Failure, + "Password grant is not permitted for this client.", + clientId, + providerName: null, + tenant, + user: null, + username: context.Request.Username, + scopes: requestedScopes, + retryAfter: null, + failureCode: AuthorityCredentialFailureCode.InvalidCredentials, + extraProperties: null); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Password grant is not permitted for this client."); + logger.LogWarning("Password grant validation failed for client {ClientId}: grant type not allowed.", clientId); + return; + } + + var allowedScopes = ClientCredentialHandlerHelpers.Split(clientDocument.Properties, AuthorityClientMetadataKeys.AllowedScopes); + var resolvedScopes = ClientCredentialHandlerHelpers.ResolveGrantedScopes(allowedScopes, requestedScopes); + + if (resolvedScopes.InvalidScope is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditInvalidScopeProperty] = resolvedScopes.InvalidScope; + + var record = PasswordGrantAuditHelper.CreatePasswordGrantRecord( + timeProvider, + context.Transaction, + metadata, + AuthEventOutcome.Failure, + $"Scope '{resolvedScopes.InvalidScope}' is not permitted for this client.", + clientId, + providerName: null, + tenant, + user: null, + username: context.Request.Username, + scopes: requestedScopes, + retryAfter: null, + failureCode: AuthorityCredentialFailureCode.InvalidCredentials, + extraProperties: null); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + + context.Reject(OpenIddictConstants.Errors.InvalidScope, $"Scope '{resolvedScopes.InvalidScope}' is not allowed for this client."); + logger.LogWarning("Password grant validation failed for client {ClientId}: scope {Scope} not permitted.", clientId, resolvedScopes.InvalidScope); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditGrantedScopesProperty] = resolvedScopes.Scopes; + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedPasswordGrantParameters(context.Request); if (unexpectedParameters.Count > 0) @@ -78,6 +207,7 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler { private readonly IAuthorityIdentityProviderRegistry registry; + private readonly IAuthorityClientStore clientStore; private readonly ActivitySource activitySource; private readonly IAuthEventSink auditSink; private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; @@ -153,6 +286,7 @@ internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler logger) { this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); @@ -192,6 +327,65 @@ internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler() : requestedScopesInput.ToArray(); + var grantedScopes = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientGrantedScopesProperty, out var grantedValue) && + grantedValue is string[] grantedArray + ? (IReadOnlyList)grantedArray + : requestedScopes; + + AuthorityClientDocument? clientDocument = null; + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTransactionProperty, out var clientValue) && + clientValue is AuthorityClientDocument storedClient) + { + clientDocument = storedClient; + } + else if (!string.IsNullOrWhiteSpace(clientId)) + { + clientDocument = await clientStore.FindByClientIdAsync(clientId, context.CancellationToken).ConfigureAwait(false); + } + + if (clientDocument is null || clientDocument.Disabled) + { + var record = PasswordGrantAuditHelper.CreatePasswordGrantRecord( + timeProvider, + context.Transaction, + metadata, + AuthEventOutcome.Failure, + "Client is not permitted for password grant.", + clientId, + providerName: null, + tenant: null, + user: null, + username: context.Request.Username, + scopes: requestedScopes, + retryAfter: null, + failureCode: AuthorityCredentialFailureCode.InvalidCredentials, + extraProperties: null); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + + context.Reject(OpenIddictConstants.Errors.InvalidClient, "The specified client is not permitted."); + logger.LogWarning("Password grant handling failed: client {ClientId} disabled or missing.", clientId); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = clientDocument; + + if (grantedScopes.Count == 0) + { + var allowedScopes = ClientCredentialHandlerHelpers.Split(clientDocument.Properties, AuthorityClientMetadataKeys.AllowedScopes); + var resolvedScopes = ClientCredentialHandlerHelpers.ResolveGrantedScopes(allowedScopes, requestedScopes); + grantedScopes = resolvedScopes.InvalidScope is null ? resolvedScopes.Scopes : Array.Empty(); + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = grantedScopes; + } + + var tenant = PasswordGrantAuditHelper.NormalizeTenant( + clientDocument.Properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var tenantValue) ? tenantValue : null); + if (!string.IsNullOrWhiteSpace(tenant)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTenantProperty] = tenant; + metadataAccessor.SetTenant(tenant); + activity?.SetTag("authority.tenant", tenant); + } var providerName = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ProviderTransactionProperty, out var value) ? value as string @@ -206,15 +400,16 @@ internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler claim.Type switch { OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken, OpenIddictConstants.Destinations.IdentityToken }, @@ -354,7 +557,7 @@ internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler? scopes, @@ -423,6 +628,7 @@ internal static class PasswordGrantAuditHelper var correlationId = EnsureCorrelationId(transaction); var normalizedScopes = NormalizeScopes(scopes); + var normalizedTenant = NormalizeTenant(tenant); var subject = BuildSubject(user, username, providerName); var client = BuildClient(clientId, providerName); var network = BuildNetwork(metadata); @@ -439,6 +645,7 @@ internal static class PasswordGrantAuditHelper Client = client, Scopes = normalizedScopes, Network = network, + Tenant = ClassifiedString.Public(normalizedTenant), Properties = properties }; } @@ -517,8 +724,9 @@ internal static class PasswordGrantAuditHelper { var remote = Normalize(metadata?.RemoteIp); var forwarded = Normalize(metadata?.ForwardedFor); + var userAgent = Normalize(metadata?.UserAgent); - if (string.IsNullOrWhiteSpace(remote) && string.IsNullOrWhiteSpace(forwarded)) + if (string.IsNullOrWhiteSpace(remote) && string.IsNullOrWhiteSpace(forwarded) && string.IsNullOrWhiteSpace(userAgent)) { return null; } @@ -526,7 +734,8 @@ internal static class PasswordGrantAuditHelper return new AuthEventNetwork { RemoteAddress = ClassifiedString.Personal(remote), - ForwardedFor = ClassifiedString.Personal(forwarded) + ForwardedFor = ClassifiedString.Personal(forwarded), + UserAgent = ClassifiedString.Personal(userAgent) }; } @@ -603,12 +812,16 @@ internal static class PasswordGrantAuditHelper private static string? Normalize(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + internal static string? NormalizeTenant(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); + internal static AuthEventRecord CreateTamperRecord( TimeProvider timeProvider, OpenIddictServerTransaction transaction, AuthorityRateLimiterMetadata? metadata, string? clientId, string? providerName, + string? tenant, string? username, IEnumerable? scopes, IEnumerable unexpectedParameters) @@ -651,6 +864,7 @@ internal static class PasswordGrantAuditHelper reason, clientId, providerName, + tenant, user: null, username, scopes, diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs index ffc79e54..df10d6d4 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs @@ -10,11 +10,12 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using OpenIddict.Abstractions; using OpenIddict.Extensions; -using OpenIddict.Server; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; -using StellaOps.Authority.Storage.Mongo.Sessions; -using StellaOps.Authority.Storage.Mongo.Stores; +using OpenIddict.Server; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Sessions; +using StellaOps.Authority.Storage.Mongo.Stores; +using StellaOps.Auth.Abstractions; namespace StellaOps.Authority.OpenIddict.Handlers; @@ -81,17 +82,23 @@ internal sealed class PersistTokensHandler : IOpenIddictServerHandler Scopes); diff --git a/src/StellaOps.Authority/StellaOps.Authority/Permalinks/VulnPermalinkService.cs b/src/StellaOps.Authority/StellaOps.Authority/Permalinks/VulnPermalinkService.cs new file mode 100644 index 00000000..c1aac64a --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Permalinks/VulnPermalinkService.cs @@ -0,0 +1,181 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Abstractions; +using StellaOps.Configuration; +using StellaOps.Cryptography; + +namespace StellaOps.Authority.Permalinks; + +internal sealed class VulnPermalinkService +{ + private static readonly JsonSerializerOptions PayloadSerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = null, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false + }; + + private static readonly TimeSpan DefaultLifetime = TimeSpan.FromHours(24); + private static readonly TimeSpan MaxLifetime = TimeSpan.FromDays(30); + private const int MaxStateBytes = 8 * 1024; + + private readonly ICryptoProviderRegistry providerRegistry; + private readonly IOptions authorityOptions; + private readonly TimeProvider timeProvider; + private readonly ILogger logger; + + public VulnPermalinkService( + ICryptoProviderRegistry providerRegistry, + IOptions authorityOptions, + TimeProvider timeProvider, + ILogger logger) + { + this.providerRegistry = providerRegistry ?? throw new ArgumentNullException(nameof(providerRegistry)); + this.authorityOptions = authorityOptions ?? throw new ArgumentNullException(nameof(authorityOptions)); + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CreateAsync(VulnPermalinkRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var tenant = request.Tenant?.Trim(); + if (string.IsNullOrWhiteSpace(tenant)) + { + throw new ArgumentException("Tenant is required.", nameof(request)); + } + + var resourceKind = request.ResourceKind?.Trim(); + if (string.IsNullOrWhiteSpace(resourceKind)) + { + throw new ArgumentException("Resource kind is required.", nameof(request)); + } + + var stateJson = request.State.ValueKind == JsonValueKind.Undefined + ? "{}" + : request.State.GetRawText(); + + if (Encoding.UTF8.GetByteCount(stateJson) > MaxStateBytes) + { + throw new ArgumentException("State payload exceeds 8 KB limit.", nameof(request)); + } + + JsonElement stateElement; + using (var stateDocument = JsonDocument.Parse(string.IsNullOrWhiteSpace(stateJson) ? "{}" : stateJson)) + { + stateElement = stateDocument.RootElement.Clone(); + } + + var lifetime = request.ExpiresInSeconds.HasValue + ? TimeSpan.FromSeconds(request.ExpiresInSeconds.Value) + : DefaultLifetime; + + if (lifetime <= TimeSpan.Zero) + { + throw new ArgumentException("Expiration must be positive.", nameof(request)); + } + + if (lifetime > MaxLifetime) + { + lifetime = MaxLifetime; + } + + var signing = authorityOptions.Value.Signing + ?? throw new InvalidOperationException("Authority signing configuration is required to issue permalinks."); + + if (!signing.Enabled) + { + throw new InvalidOperationException("Authority signing is disabled. Enable signing to issue permalinks."); + } + + if (string.IsNullOrWhiteSpace(signing.ActiveKeyId)) + { + throw new InvalidOperationException("Authority signing configuration requires an active key identifier."); + } + + var algorithm = string.IsNullOrWhiteSpace(signing.Algorithm) + ? SignatureAlgorithms.Es256 + : signing.Algorithm.Trim(); + + var issuedAt = timeProvider.GetUtcNow(); + var expiresAt = issuedAt.Add(lifetime); + + var keyReference = new CryptoKeyReference(signing.ActiveKeyId, signing.Provider); + var resolution = providerRegistry.ResolveSigner( + CryptoCapability.Signing, + algorithm, + keyReference, + signing.Provider); + var signer = resolution.Signer; + + var payload = new VulnPermalinkPayload( + Subject: "vuln:permalink", + Audience: "stellaops:vuln-explorer", + Type: resourceKind, + Tenant: tenant, + Environment: string.IsNullOrWhiteSpace(request.Environment) ? null : request.Environment.Trim(), + Scopes: new[] { StellaOpsScopes.VulnRead }, + IssuedAt: issuedAt.ToUnixTimeSeconds(), + NotBefore: issuedAt.ToUnixTimeSeconds(), + ExpiresAt: expiresAt.ToUnixTimeSeconds(), + TokenId: Guid.NewGuid().ToString("N"), + Resource: new VulnPermalinkResource(resourceKind, stateElement)); + + var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(payload, PayloadSerializerOptions); + var header = new Dictionary + { + ["alg"] = algorithm, + ["typ"] = "JWT", + ["kid"] = signer.KeyId + }; + + var headerBytes = JsonSerializer.SerializeToUtf8Bytes(header, HeaderSerializerOptions); + var encodedHeader = Base64UrlEncoder.Encode(headerBytes); + var encodedPayload = Base64UrlEncoder.Encode(payloadBytes); + + var signingInput = Encoding.ASCII.GetBytes(string.Concat(encodedHeader, '.', encodedPayload)); + var signatureBytes = await signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false); + var encodedSignature = Base64UrlEncoder.Encode(signatureBytes); + var token = string.Concat(encodedHeader, '.', encodedPayload, '.', encodedSignature); + + logger.LogDebug("Issued Vuln Explorer permalink for tenant {Tenant} with resource kind {Resource}.", tenant, resourceKind); + + return new VulnPermalinkResponse( + Token: token, + IssuedAt: issuedAt, + ExpiresAt: expiresAt, + Scopes: new[] { StellaOpsScopes.VulnRead }); + } + + private sealed record VulnPermalinkPayload( + [property: JsonPropertyName("sub")] string Subject, + [property: JsonPropertyName("aud")] string Audience, + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("tenant")] string Tenant, + [property: JsonPropertyName("environment")] string? Environment, + [property: JsonPropertyName("scopes")] IReadOnlyList Scopes, + [property: JsonPropertyName("iat")] long IssuedAt, + [property: JsonPropertyName("nbf")] long NotBefore, + [property: JsonPropertyName("exp")] long ExpiresAt, + [property: JsonPropertyName("jti")] string TokenId, + [property: JsonPropertyName("resource")] VulnPermalinkResource Resource); + + private sealed record VulnPermalinkResource( + [property: JsonPropertyName("kind")] string Kind, + [property: JsonPropertyName("state")] JsonElement State); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Program.cs b/src/StellaOps.Authority/StellaOps.Authority/Program.cs index 2c3db01e..b90ba3af 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Program.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Program.cs @@ -34,11 +34,14 @@ using StellaOps.Authority.OpenIddict.Handlers; using System.Linq; using StellaOps.Cryptography.Audit; using StellaOps.Cryptography.DependencyInjection; +using StellaOps.Authority.Permalinks; using StellaOps.Authority.Revocation; using StellaOps.Authority.Signing; using StellaOps.Cryptography; using StellaOps.Authority.Storage.Mongo.Documents; using StellaOps.Authority.Security; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; #if STELLAOPS_AUTH_SECURITY using StellaOps.Auth.Security.Dpop; using StackExchange.Redis; @@ -155,6 +158,7 @@ builder.Services.AddRateLimiter(rateLimiterOptions => builder.Services.AddStellaOpsCrypto(); builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton()); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); AuthorityPluginContext[] pluginContexts = AuthorityPluginConfigurationLoader .Load(authorityOptions, builder.Environment.ContentRootPath) @@ -228,10 +232,16 @@ builder.Services.AddOpenIddict() options.DisableAuthorizationStorage(); options.RegisterScopes( - OpenIddictConstants.Scopes.OpenId, - OpenIddictConstants.Scopes.Email, - OpenIddictConstants.Scopes.Profile, - OpenIddictConstants.Scopes.OfflineAccess); + new[] + { + OpenIddictConstants.Scopes.OpenId, + OpenIddictConstants.Scopes.Email, + OpenIddictConstants.Scopes.Profile, + OpenIddictConstants.Scopes.OfflineAccess + } + .Concat(StellaOpsScopes.All) + .Distinct(StringComparer.Ordinal) + .ToArray()); options.AddEphemeralEncryptionKey() .AddEphemeralSigningKey(); @@ -806,6 +816,9 @@ if (authorityOptions.Bootstrap.Enabled) request.AllowedAudiences ?? Array.Empty(), redirectUris, postLogoutUris, + properties.TryGetValue(AuthorityClientMetadataKeys.Tenant, out var requestedTenant) + ? requestedTenant?.Trim().ToLowerInvariant() + : null, properties, certificateBindings); @@ -1210,6 +1223,28 @@ app.MapGet("/ready", (IAuthorityIdentityProviderRegistry registry) => })) .WithName("ReadinessCheck"); +app.MapPost("/permalinks/vuln", async ( + VulnPermalinkRequest request, + VulnPermalinkService service, + CancellationToken cancellationToken) => +{ + try + { + var response = await service.CreateAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Ok(response); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { error = "invalid_request", message = ex.Message }); + } + catch (InvalidOperationException ex) + { + return Results.Problem(ex.Message); + } +}) + .RequireAuthorization(policy => policy.RequireStellaOpsScopes(StellaOpsScopes.VulnRead)) + .WithName("CreateVulnPermalink"); + app.MapGet("/jwks", (AuthorityJwksService jwksService) => Results.Ok(jwksService.Build())) .WithName("JsonWebKeySet"); diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs index 6b0dc59d..8c4faf56 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadata.cs @@ -36,6 +36,11 @@ internal sealed class AuthorityRateLimiterMetadata /// public string? SubjectId { get; set; } + /// + /// Tenant identifier associated with the request, when available. + /// + public string? Tenant { get; set; } + /// /// Additional metadata tags that can be attached by later handlers. /// diff --git a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs index ebd5e012..273a6d50 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/RateLimiting/AuthorityRateLimiterMetadataAccessor.cs @@ -24,6 +24,11 @@ internal interface IAuthorityRateLimiterMetadataAccessor /// void SetSubjectId(string? subjectId); + /// + /// Updates the tenant identifier associated with the current request. + /// + void SetTenant(string? tenant); + /// /// Adds or removes a metadata tag for the current request. /// @@ -64,6 +69,16 @@ internal sealed class AuthorityRateLimiterMetadataAccessor : IAuthorityRateLimit } } + public void SetTenant(string? tenant) + { + var metadata = TryGetMetadata(); + if (metadata is not null) + { + metadata.Tenant = NormalizeTenant(tenant); + metadata.SetTag("authority.tenant", metadata.Tenant); + } + } + public void SetTag(string name, string? value) { var metadata = TryGetMetadata(); @@ -80,4 +95,9 @@ internal sealed class AuthorityRateLimiterMetadataAccessor : IAuthorityRateLimit { return string.IsNullOrWhiteSpace(value) ? null : value; } + + private static string? NormalizeTenant(string? value) + { + return string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); + } } diff --git a/src/StellaOps.Authority/TASKS.md b/src/StellaOps.Authority/TASKS.md index b9130aa6..92f2a6cb 100644 --- a/src/StellaOps.Authority/TASKS.md +++ b/src/StellaOps.Authority/TASKS.md @@ -1,25 +1,29 @@ # Authority Host Task Board — Epic 1: Aggregation-Only Contract | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| AUTH-AOC-19-001 | TODO | Authority Core & Security Guild | — | Introduce scopes `advisory:read`, `advisory:ingest`, `vex:read`, `vex:ingest`, `aoc:verify` with configuration binding, migrations, and offline kit defaults. | Scopes published in metadata/OpenAPI, configuration validates scope lists, tests cover token issuance + enforcement. | -| AUTH-AOC-19-002 | TODO | Authority Core & Security Guild | AUTH-AOC-19-001 | Propagate tenant claim + scope enforcement for ingestion identities; ensure cross-tenant writes/read blocked and audit logs capture tenant context. | Tenant claim injected into downstream services; forbidden cross-tenant access rejected; audit/log fixtures updated. | +| AUTH-AOC-19-001 | DONE (2025-10-26) | Authority Core & Security Guild | — | Introduce scopes `advisory:read`, `advisory:ingest`, `vex:read`, `vex:ingest`, `aoc:verify` with configuration binding, migrations, and offline kit defaults. | Scopes published in metadata/OpenAPI, configuration validates scope lists, tests cover token issuance + enforcement. | +| AUTH-AOC-19-002 | DOING (2025-10-26) | Authority Core & Security Guild | AUTH-AOC-19-001 | Propagate tenant claim + scope enforcement for ingestion identities; ensure cross-tenant writes/read blocked and audit logs capture tenant context. | Tenant claim injected into downstream services; forbidden cross-tenant access rejected; audit/log fixtures updated. | +> 2025-10-26: Rate limiter metadata/audit records now include tenants, password grant scopes/tenants enforced, token persistence + tests updated. Docs refresh tracked via AUTH-AOC-19-003. | AUTH-AOC-19-003 | TODO | Authority Core & Docs Guild | AUTH-AOC-19-001 | Update Authority docs and sample configs to describe new scopes, tenancy enforcement, and verify endpoints. | Docs and examples refreshed; release notes prepared; smoke tests confirm new scopes required. | +> 2025-10-26: Docs updated (`docs/11_AUTHORITY.md`, Concelier audit runbook, `docs/security/authority-scopes.md`); sample config highlights tenant-aware clients. Release notes + smoke verification pending (blocked on Concelier/Excititor smoke updates). ## Policy Engine v2 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| AUTH-POLICY-20-001 | TODO | Authority Core & Security Guild | AUTH-AOC-19-001 | Add scopes `policy:write`, `policy:submit`, `policy:approve`, `policy:run`, `findings:read`, `effective:write` with configuration binding and issuer policy updates. | Scopes available in metadata; token issuance validated; offline kit defaults updated; tests cover scope combinations. | -| AUTH-POLICY-20-002 | TODO | Authority Core & Security Guild | AUTH-POLICY-20-001, AUTH-AOC-19-002 | Enforce Policy Engine service identity with `effective:write` and ensure API gateway enforces scopes/tenant claims for new endpoints. | Gateway policies updated; unauthorized requests rejected in tests; audit logs capture scope usage. | -| AUTH-POLICY-20-003 | TODO | Authority Core & Docs Guild | AUTH-POLICY-20-001 | Update Authority configuration/docs with policy scopes, service identities, and approval workflows; include compliance checklist. | Docs refreshed; samples updated; release notes prepared; doc lint passes. | +| AUTH-POLICY-20-001 | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-AOC-19-001 | Add scopes `policy:write`, `policy:submit`, `policy:approve`, `policy:run`, `findings:read`, `effective:write` with configuration binding and issuer policy updates. | Scopes available in metadata; token issuance validated; offline kit defaults updated; tests cover scope combinations. | +| AUTH-POLICY-20-002 | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-POLICY-20-001, AUTH-AOC-19-002 | Enforce Policy Engine service identity with `effective:write` and ensure API gateway enforces scopes/tenant claims for new endpoints. | Gateway policies updated; unauthorized requests rejected in tests; audit logs capture scope usage. | +> 2025-10-26: Restricted `effective:write` to Policy Engine service identities with tenant requirement, registered full scope set, and tightened resource server default scope enforcement (unit tests pass). +| AUTH-POLICY-20-003 | DONE (2025-10-26) | Authority Core & Docs Guild | AUTH-POLICY-20-001 | Update Authority configuration/docs with policy scopes, service identities, and approval workflows; include compliance checklist. | Docs refreshed; samples updated; release notes prepared; doc lint passes. | +> 2025-10-26: Authority docs now detail policy scopes/service identity guardrails with checklist; `authority.yaml.sample` includes `properties.serviceIdentity` example. ## Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| AUTH-GRAPH-21-001 | TODO | Authority Core & Security Guild | AUTH-POLICY-20-001 | Define scopes `graph:write`, `graph:read`, `graph:export`, `graph:simulate`, update metadata/OpenAPI, and add OFFLINE kit defaults. | Scopes exposed via discovery docs; smoke tests ensure enforcement; offline kit updated. | -| AUTH-GRAPH-21-002 | TODO | Authority Core & Security Guild | AUTH-GRAPH-21-001, AUTH-AOC-19-002 | Wire gateway enforcement for new graph scopes, Cartographer service identity, and tenant propagation across graph APIs. | Gateway config updated; unauthorized access blocked in integration tests; audit logs include graph scope usage. | -| AUTH-GRAPH-21-003 | TODO | Authority Core & Docs Guild | AUTH-GRAPH-21-001 | Update security docs and samples describing graph access roles, least privilege guidance, and service identities. | Docs merged with compliance checklist; examples refreshed; release notes prepared. | +| AUTH-GRAPH-21-001 | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-POLICY-20-001 | Define scopes `graph:write`, `graph:read`, `graph:export`, `graph:simulate`, update metadata/OpenAPI, and add OFFLINE kit defaults. | Scopes exposed via discovery docs; smoke tests ensure enforcement; offline kit updated. | +| AUTH-GRAPH-21-002 | DONE (2025-10-26) | Authority Core & Security Guild | AUTH-GRAPH-21-001, AUTH-AOC-19-002 | Wire gateway enforcement for new graph scopes, Cartographer service identity, and tenant propagation across graph APIs. | Gateway config updated; unauthorized access blocked in integration tests; audit logs include graph scope usage. | +| AUTH-GRAPH-21-003 | DONE (2025-10-26) | Authority Core & Docs Guild | AUTH-GRAPH-21-001 | Update security docs and samples describing graph access roles, least privilege guidance, and service identities. | Docs merged with compliance checklist; examples refreshed; release notes prepared. | ## Policy Engine + Editor v1 @@ -33,7 +37,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| AUTH-GRAPH-24-001 | TODO | Authority Core & Security Guild | AUTH-GRAPH-21-001 | Extend scopes to include `vuln:read` and signed permalinks with scoped claims for Graph/Vuln Explorer; update metadata. | Scopes published; permalinks validated; integration tests cover RBAC. | +| AUTH-VULN-24-001 | TODO | Authority Core & Security Guild | AUTH-GRAPH-21-001 | Extend scopes to include `vuln:read` and signed permalinks with scoped claims for Vuln Explorer; update metadata. | Scopes published; permalinks validated; integration tests cover RBAC. | ## Orchestrator Dashboard diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/README.md b/src/StellaOps.Bench/LinkNotMerge.Vex/README.md new file mode 100644 index 00000000..74efc234 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/README.md @@ -0,0 +1,26 @@ +# Link-Not-Merge VEX Bench + +Measures synthetic VEX observation ingest and event emission throughput for the Link-Not-Merge program. + +## Scenarios + +`config.json` defines workloads with varying statement density and tenant fan-out. Metrics captured per scenario: + +- Total latency (ingest + correlation) and p95/max percentiles +- Correlator-only latency and Mongo insert latency +- Observation throughput (observations/sec) +- Event emission throughput (events/sec) +- Peak managed heap allocations + +## Running locally + +```bash +dotnet run \ + --project src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.csproj \ + -- \ + --csv out/linknotmerge-vex-bench.csv \ + --json out/linknotmerge-vex-bench.json \ + --prometheus out/linknotmerge-vex-bench.prom +``` + +The benchmark exits non-zero if latency thresholds are exceeded, observation or event throughput drops below configured floors, allocations exceed the ceiling, or regression ratios breach the baseline. diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/BaselineLoaderTests.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/BaselineLoaderTests.cs new file mode 100644 index 00000000..7c00f17f --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/BaselineLoaderTests.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Bench.LinkNotMerge.Vex.Baseline; +using Xunit; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Tests; + +public sealed class BaselineLoaderTests +{ + [Fact] + public async Task LoadAsync_ReadsEntries() + { + var path = Path.GetTempFileName(); + try + { + await File.WriteAllTextAsync( + path, + "scenario,iterations,observations,statements,events,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_observation_throughput_per_sec,min_observation_throughput_per_sec,mean_event_throughput_per_sec,min_event_throughput_per_sec,max_allocated_mb\n" + + "vex_ingest_baseline,5,4000,24000,12000,620.5,700.1,820.9,320.5,300.0,9800.0,9100.0,4200.0,3900.0,150.0\n"); + + var baseline = await BaselineLoader.LoadAsync(path, CancellationToken.None); + var entry = Assert.Single(baseline); + + Assert.Equal("vex_ingest_baseline", entry.Key); + Assert.Equal(4000, entry.Value.Observations); + Assert.Equal(24000, entry.Value.Statements); + Assert.Equal(12000, entry.Value.Events); + Assert.Equal(700.1, entry.Value.P95TotalMs); + Assert.Equal(3900.0, entry.Value.MinEventThroughputPerSecond); + } + finally + { + File.Delete(path); + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/BenchmarkScenarioReportTests.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/BenchmarkScenarioReportTests.cs new file mode 100644 index 00000000..2f4064ef --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/BenchmarkScenarioReportTests.cs @@ -0,0 +1,83 @@ +using StellaOps.Bench.LinkNotMerge.Vex.Baseline; +using StellaOps.Bench.LinkNotMerge.Vex.Reporting; +using Xunit; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Tests; + +public sealed class BenchmarkScenarioReportTests +{ + [Fact] + public void RegressionDetection_FlagsBreaches() + { + var result = new VexScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + ObservationCount: 1000, + AliasGroups: 100, + StatementCount: 6000, + EventCount: 3200, + TotalStatistics: new DurationStatistics(600, 700, 750), + InsertStatistics: new DurationStatistics(320, 360, 380), + CorrelationStatistics: new DurationStatistics(280, 320, 340), + ObservationThroughputStatistics: new ThroughputStatistics(8000, 7000), + EventThroughputStatistics: new ThroughputStatistics(3500, 3200), + AllocationStatistics: new AllocationStatistics(180), + ThresholdMs: null, + MinObservationThroughputPerSecond: null, + MinEventThroughputPerSecond: null, + MaxAllocatedThresholdMb: null); + + var baseline = new BaselineEntry( + ScenarioId: "scenario", + Iterations: 3, + Observations: 1000, + Statements: 6000, + Events: 3200, + MeanTotalMs: 520, + P95TotalMs: 560, + MaxTotalMs: 580, + MeanInsertMs: 250, + MeanCorrelationMs: 260, + MeanObservationThroughputPerSecond: 9000, + MinObservationThroughputPerSecond: 8500, + MeanEventThroughputPerSecond: 4200, + MinEventThroughputPerSecond: 3800, + MaxAllocatedMb: 140); + + var report = new BenchmarkScenarioReport(result, baseline, regressionLimit: 1.1); + + Assert.True(report.DurationRegressionBreached); + Assert.True(report.ObservationThroughputRegressionBreached); + Assert.True(report.EventThroughputRegressionBreached); + Assert.Contains(report.BuildRegressionFailureMessages(), message => message.Contains("event throughput")); + } + + [Fact] + public void RegressionDetection_NoBaseline_NoBreaches() + { + var result = new VexScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + ObservationCount: 1000, + AliasGroups: 100, + StatementCount: 6000, + EventCount: 3200, + TotalStatistics: new DurationStatistics(480, 520, 540), + InsertStatistics: new DurationStatistics(260, 280, 300), + CorrelationStatistics: new DurationStatistics(220, 240, 260), + ObservationThroughputStatistics: new ThroughputStatistics(9000, 8800), + EventThroughputStatistics: new ThroughputStatistics(4200, 4100), + AllocationStatistics: new AllocationStatistics(150), + ThresholdMs: null, + MinObservationThroughputPerSecond: null, + MinEventThroughputPerSecond: null, + MaxAllocatedThresholdMb: null); + + var report = new BenchmarkScenarioReport(result, baseline: null, regressionLimit: null); + + Assert.False(report.RegressionBreached); + Assert.Empty(report.BuildRegressionFailureMessages()); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/StellaOps.Bench.LinkNotMerge.Vex.Tests.csproj b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/StellaOps.Bench.LinkNotMerge.Vex.Tests.csproj new file mode 100644 index 00000000..f734d08a --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/StellaOps.Bench.LinkNotMerge.Vex.Tests.csproj @@ -0,0 +1,28 @@ + + + net10.0 + enable + enable + preview + true + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/VexScenarioRunnerTests.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/VexScenarioRunnerTests.cs new file mode 100644 index 00000000..6858203f --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/VexScenarioRunnerTests.cs @@ -0,0 +1,34 @@ +using System.Linq; +using System.Threading; +using Xunit; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Tests; + +public sealed class VexScenarioRunnerTests +{ + [Fact] + public void Execute_ComputesEvents() + { + var config = new VexScenarioConfig + { + Id = "unit", + Observations = 600, + AliasGroups = 120, + StatementsPerObservation = 5, + ProductsPerObservation = 3, + Tenants = 2, + BatchSize = 120, + Seed = 12345, + }; + + var runner = new VexScenarioRunner(config); + var result = runner.Execute(2, CancellationToken.None); + + Assert.Equal(600, result.ObservationCount); + Assert.True(result.StatementCount > 0); + Assert.True(result.EventCount > 0); + Assert.All(result.TotalDurationsMs, duration => Assert.True(duration > 0)); + Assert.All(result.EventThroughputsPerSecond, throughput => Assert.True(throughput > 0)); + Assert.Equal(result.AggregationResult.EventCount, result.EventCount); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Baseline/BaselineEntry.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Baseline/BaselineEntry.cs new file mode 100644 index 00000000..5c53af34 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Baseline/BaselineEntry.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Bench.LinkNotMerge.Vex.Baseline; + +internal sealed record BaselineEntry( + string ScenarioId, + int Iterations, + int Observations, + int Statements, + int Events, + double MeanTotalMs, + double P95TotalMs, + double MaxTotalMs, + double MeanInsertMs, + double MeanCorrelationMs, + double MeanObservationThroughputPerSecond, + double MinObservationThroughputPerSecond, + double MeanEventThroughputPerSecond, + double MinEventThroughputPerSecond, + double MaxAllocatedMb); diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Baseline/BaselineLoader.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Baseline/BaselineLoader.cs new file mode 100644 index 00000000..a541384a --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Baseline/BaselineLoader.cs @@ -0,0 +1,87 @@ +using System.Globalization; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Baseline; + +internal static class BaselineLoader +{ + public static async Task> LoadAsync(string path, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read); + using var reader = new StreamReader(stream); + + var lineNumber = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var line = await reader.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + lineNumber++; + if (lineNumber == 1 || string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = line.Split(',', StringSplitOptions.TrimEntries); + if (parts.Length < 15) + { + throw new InvalidOperationException($"Baseline '{resolved}' line {lineNumber} is invalid (expected 15 columns, found {parts.Length})."); + } + + var entry = new BaselineEntry( + ScenarioId: parts[0], + Iterations: ParseInt(parts[1], resolved, lineNumber), + Observations: ParseInt(parts[2], resolved, lineNumber), + Statements: ParseInt(parts[3], resolved, lineNumber), + Events: ParseInt(parts[4], resolved, lineNumber), + MeanTotalMs: ParseDouble(parts[5], resolved, lineNumber), + P95TotalMs: ParseDouble(parts[6], resolved, lineNumber), + MaxTotalMs: ParseDouble(parts[7], resolved, lineNumber), + MeanInsertMs: ParseDouble(parts[8], resolved, lineNumber), + MeanCorrelationMs: ParseDouble(parts[9], resolved, lineNumber), + MeanObservationThroughputPerSecond: ParseDouble(parts[10], resolved, lineNumber), + MinObservationThroughputPerSecond: ParseDouble(parts[11], resolved, lineNumber), + MeanEventThroughputPerSecond: ParseDouble(parts[12], resolved, lineNumber), + MinEventThroughputPerSecond: ParseDouble(parts[13], resolved, lineNumber), + MaxAllocatedMb: ParseDouble(parts[14], resolved, lineNumber)); + + result[entry.ScenarioId] = entry; + } + + return result; + } + + private static int ParseInt(string value, string file, int line) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid integer '{value}'."); + } + + private static double ParseDouble(string value, string file, int line) + { + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid number '{value}'."); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Program.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Program.cs new file mode 100644 index 00000000..e70d359a --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Program.cs @@ -0,0 +1,376 @@ +using System.Globalization; +using StellaOps.Bench.LinkNotMerge.Vex.Baseline; +using StellaOps.Bench.LinkNotMerge.Vex.Reporting; + +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal static class Program +{ + public static async Task Main(string[] args) + { + try + { + var options = ProgramOptions.Parse(args); + var config = await VexBenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false); + var baseline = await BaselineLoader.LoadAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false); + + var results = new List(); + var reports = new List(); + var failures = new List(); + + foreach (var scenario in config.Scenarios) + { + var iterations = scenario.ResolveIterations(config.Iterations); + var runner = new VexScenarioRunner(scenario); + var execution = runner.Execute(iterations, CancellationToken.None); + + var totalStats = DurationStatistics.From(execution.TotalDurationsMs); + var insertStats = DurationStatistics.From(execution.InsertDurationsMs); + var correlationStats = DurationStatistics.From(execution.CorrelationDurationsMs); + var allocationStats = AllocationStatistics.From(execution.AllocatedMb); + var observationThroughputStats = ThroughputStatistics.From(execution.ObservationThroughputsPerSecond); + var eventThroughputStats = ThroughputStatistics.From(execution.EventThroughputsPerSecond); + + var thresholdMs = scenario.ThresholdMs ?? options.ThresholdMs ?? config.ThresholdMs; + var observationFloor = scenario.MinThroughputPerSecond ?? options.MinThroughputPerSecond ?? config.MinThroughputPerSecond; + var eventFloor = scenario.MinEventThroughputPerSecond ?? options.MinEventThroughputPerSecond ?? config.MinEventThroughputPerSecond; + var allocationLimit = scenario.MaxAllocatedMb ?? options.MaxAllocatedMb ?? config.MaxAllocatedMb; + + var result = new VexScenarioResult( + scenario.ScenarioId, + scenario.DisplayLabel, + iterations, + execution.ObservationCount, + execution.AliasGroups, + execution.StatementCount, + execution.EventCount, + totalStats, + insertStats, + correlationStats, + observationThroughputStats, + eventThroughputStats, + allocationStats, + thresholdMs, + observationFloor, + eventFloor, + allocationLimit); + + results.Add(result); + + if (thresholdMs is { } threshold && result.TotalStatistics.MaxMs > threshold) + { + failures.Add($"{result.Id} exceeded total latency threshold: {result.TotalStatistics.MaxMs:F2} ms > {threshold:F2} ms"); + } + + if (observationFloor is { } obsFloor && result.ObservationThroughputStatistics.MinPerSecond < obsFloor) + { + failures.Add($"{result.Id} fell below observation throughput floor: {result.ObservationThroughputStatistics.MinPerSecond:N0} obs/s < {obsFloor:N0} obs/s"); + } + + if (eventFloor is { } evtFloor && result.EventThroughputStatistics.MinPerSecond < evtFloor) + { + failures.Add($"{result.Id} fell below event throughput floor: {result.EventThroughputStatistics.MinPerSecond:N0} events/s < {evtFloor:N0} events/s"); + } + + if (allocationLimit is { } limit && result.AllocationStatistics.MaxAllocatedMb > limit) + { + failures.Add($"{result.Id} exceeded allocation budget: {result.AllocationStatistics.MaxAllocatedMb:F2} MB > {limit:F2} MB"); + } + + baseline.TryGetValue(result.Id, out var baselineEntry); + var report = new BenchmarkScenarioReport(result, baselineEntry, options.RegressionLimit); + reports.Add(report); + failures.AddRange(report.BuildRegressionFailureMessages()); + } + + TablePrinter.Print(results); + + if (!string.IsNullOrWhiteSpace(options.CsvOutPath)) + { + CsvWriter.Write(options.CsvOutPath!, results); + } + + if (!string.IsNullOrWhiteSpace(options.JsonOutPath)) + { + var metadata = new BenchmarkJsonMetadata( + SchemaVersion: "linknotmerge-vex-bench/1.0", + CapturedAtUtc: (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime(), + Commit: options.Commit, + Environment: options.Environment); + + await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath)) + { + PrometheusWriter.Write(options.PrometheusOutPath!, reports); + } + + if (failures.Count > 0) + { + Console.Error.WriteLine(); + Console.Error.WriteLine("Benchmark failures detected:"); + foreach (var failure in failures.Distinct()) + { + Console.Error.WriteLine($" - {failure}"); + } + + return 1; + } + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"linknotmerge-vex-bench error: {ex.Message}"); + return 1; + } + } + + private sealed record ProgramOptions( + string ConfigPath, + int? Iterations, + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MinEventThroughputPerSecond, + double? MaxAllocatedMb, + string? CsvOutPath, + string? JsonOutPath, + string? PrometheusOutPath, + string BaselinePath, + DateTimeOffset? CapturedAtUtc, + string? Commit, + string? Environment, + double? RegressionLimit) + { + public static ProgramOptions Parse(string[] args) + { + var configPath = DefaultConfigPath(); + var baselinePath = DefaultBaselinePath(); + + int? iterations = null; + double? thresholdMs = null; + double? minThroughput = null; + double? minEventThroughput = null; + double? maxAllocated = null; + string? csvOut = null; + string? jsonOut = null; + string? promOut = null; + DateTimeOffset? capturedAt = null; + string? commit = null; + string? environment = null; + double? regressionLimit = null; + + for (var index = 0; index < args.Length; index++) + { + var current = args[index]; + switch (current) + { + case "--config": + EnsureNext(args, index); + configPath = Path.GetFullPath(args[++index]); + break; + case "--iterations": + EnsureNext(args, index); + iterations = int.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--threshold-ms": + EnsureNext(args, index); + thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--min-throughput": + EnsureNext(args, index); + minThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--min-event-throughput": + EnsureNext(args, index); + minEventThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--max-allocated-mb": + EnsureNext(args, index); + maxAllocated = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--csv": + EnsureNext(args, index); + csvOut = args[++index]; + break; + case "--json": + EnsureNext(args, index); + jsonOut = args[++index]; + break; + case "--prometheus": + EnsureNext(args, index); + promOut = args[++index]; + break; + case "--baseline": + EnsureNext(args, index); + baselinePath = Path.GetFullPath(args[++index]); + break; + case "--captured-at": + EnsureNext(args, index); + capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); + break; + case "--commit": + EnsureNext(args, index); + commit = args[++index]; + break; + case "--environment": + EnsureNext(args, index); + environment = args[++index]; + break; + case "--regression-limit": + EnsureNext(args, index); + regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--help": + case "-h": + PrintUsage(); + System.Environment.Exit(0); + break; + default: + throw new ArgumentException($"Unknown argument '{current}'."); + } + } + + return new ProgramOptions( + configPath, + iterations, + thresholdMs, + minThroughput, + minEventThroughput, + maxAllocated, + csvOut, + jsonOut, + promOut, + baselinePath, + capturedAt, + commit, + environment, + regressionLimit); + } + + private static string DefaultConfigPath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "config.json"); + } + + private static string DefaultBaselinePath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "baseline.csv"); + } + + private static void EnsureNext(string[] args, int index) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException("Missing value for argument."); + } + } + + private static void PrintUsage() + { + Console.WriteLine("Usage: linknotmerge-vex-bench [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --config Path to benchmark configuration JSON."); + Console.WriteLine(" --iterations Override iteration count."); + Console.WriteLine(" --threshold-ms Global latency threshold in milliseconds."); + Console.WriteLine(" --min-throughput Observation throughput floor (observations/second)."); + Console.WriteLine(" --min-event-throughput Event emission throughput floor (events/second)."); + Console.WriteLine(" --max-allocated-mb Global allocation ceiling (MB)."); + Console.WriteLine(" --csv Write CSV results to path."); + Console.WriteLine(" --json Write JSON results to path."); + Console.WriteLine(" --prometheus Write Prometheus exposition metrics to path."); + Console.WriteLine(" --baseline Baseline CSV path."); + Console.WriteLine(" --captured-at Timestamp to embed in JSON metadata."); + Console.WriteLine(" --commit Commit identifier for metadata."); + Console.WriteLine(" --environment Environment label for metadata."); + Console.WriteLine(" --regression-limit Regression multiplier (default 1.15)."); + } + } +} + +internal static class TablePrinter +{ + public static void Print(IEnumerable results) + { + Console.WriteLine("Scenario | Observations | Statements | Events | Total(ms) | Correl(ms) | Insert(ms) | Obs k/s | Evnt k/s | Alloc(MB)"); + Console.WriteLine("---------------------------- | ------------- | ---------- | ------- | ---------- | ---------- | ----------- | ------- | -------- | --------"); + foreach (var row in results) + { + Console.WriteLine(string.Join(" | ", new[] + { + row.IdColumn, + row.ObservationsColumn, + row.StatementColumn, + row.EventColumn, + row.TotalMeanColumn, + row.CorrelationMeanColumn, + row.InsertMeanColumn, + row.ObservationThroughputColumn, + row.EventThroughputColumn, + row.AllocatedColumn, + })); + } + } +} + +internal static class CsvWriter +{ + public static void Write(string path, IEnumerable results) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(results); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(stream); + writer.WriteLine("scenario,iterations,observations,statements,events,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_observation_throughput_per_sec,min_observation_throughput_per_sec,mean_event_throughput_per_sec,min_event_throughput_per_sec,max_allocated_mb"); + + foreach (var result in results) + { + writer.Write(result.Id); + writer.Write(','); + writer.Write(result.Iterations.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.ObservationCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.StatementCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.EventCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalStatistics.P95Ms.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalStatistics.MaxMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.InsertStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.CorrelationStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.ObservationThroughputStatistics.MeanPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.ObservationThroughputStatistics.MinPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.EventThroughputStatistics.MeanPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.EventThroughputStatistics.MinPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.AllocationStatistics.MaxAllocatedMb.ToString("F4", CultureInfo.InvariantCulture)); + writer.WriteLine(); + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Properties/AssemblyInfo.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..cb24d2c2 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Bench.LinkNotMerge.Vex.Tests")] diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/BenchmarkJsonWriter.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/BenchmarkJsonWriter.cs new file mode 100644 index 00000000..89d04b43 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/BenchmarkJsonWriter.cs @@ -0,0 +1,151 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Reporting; + +internal static class BenchmarkJsonWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public static async Task WriteAsync( + string path, + BenchmarkJsonMetadata metadata, + IReadOnlyList reports, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(metadata); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var document = new BenchmarkJsonDocument( + metadata.SchemaVersion, + metadata.CapturedAtUtc, + metadata.Commit, + metadata.Environment, + reports.Select(CreateScenario).ToArray()); + + await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report) + { + var baseline = report.Baseline; + return new BenchmarkJsonScenario( + report.Result.Id, + report.Result.Label, + report.Result.Iterations, + report.Result.ObservationCount, + report.Result.StatementCount, + report.Result.EventCount, + report.Result.TotalStatistics.MeanMs, + report.Result.TotalStatistics.P95Ms, + report.Result.TotalStatistics.MaxMs, + report.Result.InsertStatistics.MeanMs, + report.Result.CorrelationStatistics.MeanMs, + report.Result.ObservationThroughputStatistics.MeanPerSecond, + report.Result.ObservationThroughputStatistics.MinPerSecond, + report.Result.EventThroughputStatistics.MeanPerSecond, + report.Result.EventThroughputStatistics.MinPerSecond, + report.Result.AllocationStatistics.MaxAllocatedMb, + report.Result.ThresholdMs, + report.Result.MinObservationThroughputPerSecond, + report.Result.MinEventThroughputPerSecond, + report.Result.MaxAllocatedThresholdMb, + baseline is null + ? null + : new BenchmarkJsonScenarioBaseline( + baseline.Iterations, + baseline.Observations, + baseline.Statements, + baseline.Events, + baseline.MeanTotalMs, + baseline.P95TotalMs, + baseline.MaxTotalMs, + baseline.MeanInsertMs, + baseline.MeanCorrelationMs, + baseline.MeanObservationThroughputPerSecond, + baseline.MinObservationThroughputPerSecond, + baseline.MeanEventThroughputPerSecond, + baseline.MinEventThroughputPerSecond, + baseline.MaxAllocatedMb), + new BenchmarkJsonScenarioRegression( + report.DurationRegressionRatio, + report.ObservationThroughputRegressionRatio, + report.EventThroughputRegressionRatio, + report.RegressionLimit, + report.RegressionBreached)); + } + + private sealed record BenchmarkJsonDocument( + string SchemaVersion, + DateTimeOffset CapturedAt, + string? Commit, + string? Environment, + IReadOnlyList Scenarios); + + private sealed record BenchmarkJsonScenario( + string Id, + string Label, + int Iterations, + int Observations, + int Statements, + int Events, + double MeanTotalMs, + double P95TotalMs, + double MaxTotalMs, + double MeanInsertMs, + double MeanCorrelationMs, + double MeanObservationThroughputPerSecond, + double MinObservationThroughputPerSecond, + double MeanEventThroughputPerSecond, + double MinEventThroughputPerSecond, + double MaxAllocatedMb, + double? ThresholdMs, + double? MinObservationThroughputThresholdPerSecond, + double? MinEventThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb, + BenchmarkJsonScenarioBaseline? Baseline, + BenchmarkJsonScenarioRegression Regression); + + private sealed record BenchmarkJsonScenarioBaseline( + int Iterations, + int Observations, + int Statements, + int Events, + double MeanTotalMs, + double P95TotalMs, + double MaxTotalMs, + double MeanInsertMs, + double MeanCorrelationMs, + double MeanObservationThroughputPerSecond, + double MinObservationThroughputPerSecond, + double MeanEventThroughputPerSecond, + double MinEventThroughputPerSecond, + double MaxAllocatedMb); + + private sealed record BenchmarkJsonScenarioRegression( + double? DurationRatio, + double? ObservationThroughputRatio, + double? EventThroughputRatio, + double Limit, + bool Breached); +} + +internal sealed record BenchmarkJsonMetadata( + string SchemaVersion, + DateTimeOffset CapturedAtUtc, + string? Commit, + string? Environment); diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/BenchmarkScenarioReport.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/BenchmarkScenarioReport.cs new file mode 100644 index 00000000..cc933b4e --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/BenchmarkScenarioReport.cs @@ -0,0 +1,89 @@ +using StellaOps.Bench.LinkNotMerge.Vex.Baseline; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Reporting; + +internal sealed class BenchmarkScenarioReport +{ + private const double DefaultRegressionLimit = 1.15d; + + public BenchmarkScenarioReport(VexScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null) + { + Result = result ?? throw new ArgumentNullException(nameof(result)); + Baseline = baseline; + RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : DefaultRegressionLimit; + DurationRegressionRatio = CalculateRatio(result.TotalStatistics.MaxMs, baseline?.MaxTotalMs); + ObservationThroughputRegressionRatio = CalculateInverseRatio(result.ObservationThroughputStatistics.MinPerSecond, baseline?.MinObservationThroughputPerSecond); + EventThroughputRegressionRatio = CalculateInverseRatio(result.EventThroughputStatistics.MinPerSecond, baseline?.MinEventThroughputPerSecond); + } + + public VexScenarioResult Result { get; } + + public BaselineEntry? Baseline { get; } + + public double RegressionLimit { get; } + + public double? DurationRegressionRatio { get; } + + public double? ObservationThroughputRegressionRatio { get; } + + public double? EventThroughputRegressionRatio { get; } + + public bool DurationRegressionBreached => DurationRegressionRatio is { } ratio && ratio >= RegressionLimit; + + public bool ObservationThroughputRegressionBreached => ObservationThroughputRegressionRatio is { } ratio && ratio >= RegressionLimit; + + public bool EventThroughputRegressionBreached => EventThroughputRegressionRatio is { } ratio && ratio >= RegressionLimit; + + public bool RegressionBreached => DurationRegressionBreached || ObservationThroughputRegressionBreached || EventThroughputRegressionBreached; + + public IEnumerable BuildRegressionFailureMessages() + { + if (Baseline is null) + { + yield break; + } + + if (DurationRegressionBreached && DurationRegressionRatio is { } durationRatio) + { + var delta = (durationRatio - 1d) * 100d; + yield return $"{Result.Id} exceeded max duration budget: {Result.TotalStatistics.MaxMs:F2} ms vs baseline {Baseline.MaxTotalMs:F2} ms (+{delta:F1}%)."; + } + + if (ObservationThroughputRegressionBreached && ObservationThroughputRegressionRatio is { } obsRatio) + { + var delta = (obsRatio - 1d) * 100d; + yield return $"{Result.Id} observation throughput regressed: min {Result.ObservationThroughputStatistics.MinPerSecond:N0} obs/s vs baseline {Baseline.MinObservationThroughputPerSecond:N0} obs/s (-{delta:F1}%)."; + } + + if (EventThroughputRegressionBreached && EventThroughputRegressionRatio is { } evtRatio) + { + var delta = (evtRatio - 1d) * 100d; + yield return $"{Result.Id} event throughput regressed: min {Result.EventThroughputStatistics.MinPerSecond:N0} events/s vs baseline {Baseline.MinEventThroughputPerSecond:N0} events/s (-{delta:F1}%)."; + } + } + + private static double? CalculateRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + return current / baseline.Value; + } + + private static double? CalculateInverseRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + if (current <= 0d) + { + return double.PositiveInfinity; + } + + return baseline.Value / current; + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/PrometheusWriter.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/PrometheusWriter.cs new file mode 100644 index 00000000..c8c2adfb --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Reporting/PrometheusWriter.cs @@ -0,0 +1,94 @@ +using System.Globalization; +using System.Text; + +namespace StellaOps.Bench.LinkNotMerge.Vex.Reporting; + +internal static class PrometheusWriter +{ + public static void Write(string path, IReadOnlyList reports) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var builder = new StringBuilder(); + builder.AppendLine("# HELP linknotmerge_vex_bench_total_ms Link-Not-Merge VEX benchmark total duration (milliseconds)."); + builder.AppendLine("# TYPE linknotmerge_vex_bench_total_ms gauge"); + builder.AppendLine("# HELP linknotmerge_vex_bench_throughput_per_sec Link-Not-Merge VEX benchmark observation throughput (observations per second)."); + builder.AppendLine("# TYPE linknotmerge_vex_bench_throughput_per_sec gauge"); + builder.AppendLine("# HELP linknotmerge_vex_bench_event_throughput_per_sec Link-Not-Merge VEX benchmark event throughput (events per second)."); + builder.AppendLine("# TYPE linknotmerge_vex_bench_event_throughput_per_sec gauge"); + builder.AppendLine("# HELP linknotmerge_vex_bench_allocated_mb Link-Not-Merge VEX benchmark max allocations (megabytes)."); + builder.AppendLine("# TYPE linknotmerge_vex_bench_allocated_mb gauge"); + + foreach (var report in reports) + { + var scenario = Escape(report.Result.Id); + AppendMetric(builder, "linknotmerge_vex_bench_mean_total_ms", scenario, report.Result.TotalStatistics.MeanMs); + AppendMetric(builder, "linknotmerge_vex_bench_p95_total_ms", scenario, report.Result.TotalStatistics.P95Ms); + AppendMetric(builder, "linknotmerge_vex_bench_max_total_ms", scenario, report.Result.TotalStatistics.MaxMs); + AppendMetric(builder, "linknotmerge_vex_bench_threshold_ms", scenario, report.Result.ThresholdMs); + + AppendMetric(builder, "linknotmerge_vex_bench_mean_observation_throughput_per_sec", scenario, report.Result.ObservationThroughputStatistics.MeanPerSecond); + AppendMetric(builder, "linknotmerge_vex_bench_min_observation_throughput_per_sec", scenario, report.Result.ObservationThroughputStatistics.MinPerSecond); + AppendMetric(builder, "linknotmerge_vex_bench_observation_throughput_floor_per_sec", scenario, report.Result.MinObservationThroughputPerSecond); + + AppendMetric(builder, "linknotmerge_vex_bench_mean_event_throughput_per_sec", scenario, report.Result.EventThroughputStatistics.MeanPerSecond); + AppendMetric(builder, "linknotmerge_vex_bench_min_event_throughput_per_sec", scenario, report.Result.EventThroughputStatistics.MinPerSecond); + AppendMetric(builder, "linknotmerge_vex_bench_event_throughput_floor_per_sec", scenario, report.Result.MinEventThroughputPerSecond); + + AppendMetric(builder, "linknotmerge_vex_bench_max_allocated_mb", scenario, report.Result.AllocationStatistics.MaxAllocatedMb); + AppendMetric(builder, "linknotmerge_vex_bench_max_allocated_threshold_mb", scenario, report.Result.MaxAllocatedThresholdMb); + + if (report.Baseline is { } baseline) + { + AppendMetric(builder, "linknotmerge_vex_bench_baseline_max_total_ms", scenario, baseline.MaxTotalMs); + AppendMetric(builder, "linknotmerge_vex_bench_baseline_min_observation_throughput_per_sec", scenario, baseline.MinObservationThroughputPerSecond); + AppendMetric(builder, "linknotmerge_vex_bench_baseline_min_event_throughput_per_sec", scenario, baseline.MinEventThroughputPerSecond); + } + + if (report.DurationRegressionRatio is { } durationRatio) + { + AppendMetric(builder, "linknotmerge_vex_bench_duration_regression_ratio", scenario, durationRatio); + } + + if (report.ObservationThroughputRegressionRatio is { } obsRatio) + { + AppendMetric(builder, "linknotmerge_vex_bench_observation_regression_ratio", scenario, obsRatio); + } + + if (report.EventThroughputRegressionRatio is { } evtRatio) + { + AppendMetric(builder, "linknotmerge_vex_bench_event_regression_ratio", scenario, evtRatio); + } + + AppendMetric(builder, "linknotmerge_vex_bench_regression_limit", scenario, report.RegressionLimit); + AppendMetric(builder, "linknotmerge_vex_bench_regression_breached", scenario, report.RegressionBreached ? 1 : 0); + } + + File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8); + } + + private static void AppendMetric(StringBuilder builder, string metric, string scenario, double? value) + { + if (!value.HasValue) + { + return; + } + + builder.Append(metric); + builder.Append("{scenario=\""); + builder.Append(scenario); + builder.Append("\"} "); + builder.AppendLine(value.Value.ToString("G17", CultureInfo.InvariantCulture)); + } + + private static string Escape(string value) => + value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Statistics.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Statistics.cs new file mode 100644 index 00000000..a9277c97 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/Statistics.cs @@ -0,0 +1,84 @@ +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal readonly record struct DurationStatistics(double MeanMs, double P95Ms, double MaxMs) +{ + public static DurationStatistics From(IReadOnlyList values) + { + if (values.Count == 0) + { + return new DurationStatistics(0, 0, 0); + } + + var sorted = values.ToArray(); + Array.Sort(sorted); + + var total = 0d; + foreach (var value in values) + { + total += value; + } + + var mean = total / values.Count; + var p95 = Percentile(sorted, 95); + var max = sorted[^1]; + + return new DurationStatistics(mean, p95, max); + } + + private static double Percentile(IReadOnlyList sorted, double percentile) + { + if (sorted.Count == 0) + { + return 0; + } + + var rank = (percentile / 100d) * (sorted.Count - 1); + var lower = (int)Math.Floor(rank); + var upper = (int)Math.Ceiling(rank); + var weight = rank - lower; + + if (upper >= sorted.Count) + { + return sorted[lower]; + } + + return sorted[lower] + weight * (sorted[upper] - sorted[lower]); + } +} + +internal readonly record struct ThroughputStatistics(double MeanPerSecond, double MinPerSecond) +{ + public static ThroughputStatistics From(IReadOnlyList values) + { + if (values.Count == 0) + { + return new ThroughputStatistics(0, 0); + } + + var total = 0d; + var min = double.MaxValue; + + foreach (var value in values) + { + total += value; + min = Math.Min(min, value); + } + + var mean = total / values.Count; + return new ThroughputStatistics(mean, min); + } +} + +internal readonly record struct AllocationStatistics(double MaxAllocatedMb) +{ + public static AllocationStatistics From(IReadOnlyList values) + { + var max = 0d; + foreach (var value in values) + { + max = Math.Max(max, value); + } + + return new AllocationStatistics(max); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.csproj b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.csproj new file mode 100644 index 00000000..7167d07e --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.csproj @@ -0,0 +1,16 @@ + + + Exe + net10.0 + enable + enable + preview + true + + + + + + + + diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexLinksetAggregator.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexLinksetAggregator.cs new file mode 100644 index 00000000..d2349087 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexLinksetAggregator.cs @@ -0,0 +1,166 @@ +using MongoDB.Bson; + +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal sealed class VexLinksetAggregator +{ + public VexAggregationResult Correlate(IEnumerable documents) + { + ArgumentNullException.ThrowIfNull(documents); + + var groups = new Dictionary(StringComparer.Ordinal); + var statementsSeen = 0; + + foreach (var document in documents) + { + var tenant = document.GetValue("tenant", "unknown").AsString; + var linksetValue = document.GetValue("linkset", new BsonDocument()); + var linkset = linksetValue.IsBsonDocument ? linksetValue.AsBsonDocument : new BsonDocument(); + var aliases = linkset.GetValue("aliases", new BsonArray()).AsBsonArray; + + var statementsValue = document.GetValue("statements", new BsonArray()); + var statements = statementsValue.IsBsonArray ? statementsValue.AsBsonArray : new BsonArray(); + + foreach (var statementValue in statements) + { + if (!statementValue.IsBsonDocument) + { + continue; + } + + statementsSeen++; + + var statement = statementValue.AsBsonDocument; + var status = statement.GetValue("status", "unknown").AsString; + var justification = statement.GetValue("justification", BsonNull.Value); + var lastUpdated = statement.GetValue("last_updated", BsonNull.Value); + var productValue = statement.GetValue("product", new BsonDocument()); + var product = productValue.IsBsonDocument ? productValue.AsBsonDocument : new BsonDocument(); + var productKey = product.GetValue("purl", "unknown").AsString; + + foreach (var aliasValue in aliases) + { + if (!aliasValue.IsString) + { + continue; + } + + var alias = aliasValue.AsString; + var key = string.Create(alias.Length + tenant.Length + productKey.Length + 2, (tenant, alias, productKey), static (span, data) => + { + var (tenantValue, aliasValue, productValue) = data; + var offset = 0; + tenantValue.AsSpan().CopyTo(span); + offset += tenantValue.Length; + span[offset++] = '|'; + aliasValue.AsSpan().CopyTo(span[offset..]); + offset += aliasValue.Length; + span[offset++] = '|'; + productValue.AsSpan().CopyTo(span[offset..]); + }); + + if (!groups.TryGetValue(key, out var accumulator)) + { + accumulator = new VexAccumulator(tenant, alias, productKey); + groups[key] = accumulator; + } + + accumulator.AddStatement(status, justification, lastUpdated); + } + } + } + + var eventDocuments = new List(groups.Count); + foreach (var accumulator in groups.Values) + { + if (accumulator.ShouldEmitEvent) + { + eventDocuments.Add(accumulator.ToEvent()); + } + } + + return new VexAggregationResult( + LinksetCount: groups.Count, + StatementCount: statementsSeen, + EventCount: eventDocuments.Count, + EventDocuments: eventDocuments); + } + + private sealed class VexAccumulator + { + private readonly Dictionary _statusCounts = new(StringComparer.Ordinal); + private readonly HashSet _justifications = new(StringComparer.Ordinal); + private readonly string _tenant; + private readonly string _alias; + private readonly string _product; + private DateTime? _latest; + + public VexAccumulator(string tenant, string alias, string product) + { + _tenant = tenant; + _alias = alias; + _product = product; + } + + public void AddStatement(string status, BsonValue justification, BsonValue updatedAt) + { + if (!_statusCounts.TryAdd(status, 1)) + { + _statusCounts[status]++; + } + + if (justification.IsString) + { + _justifications.Add(justification.AsString); + } + + if (updatedAt.IsValidDateTime) + { + var value = updatedAt.ToUniversalTime(); + if (!_latest.HasValue || value > _latest) + { + _latest = value; + } + } + } + + public bool ShouldEmitEvent + { + get + { + if (_statusCounts.TryGetValue("affected", out var affected) && affected > 0) + { + return true; + } + + if (_statusCounts.TryGetValue("under_investigation", out var investigating) && investigating > 0) + { + return true; + } + + return false; + } + } + + public BsonDocument ToEvent() + { + var payload = new BsonDocument + { + ["tenant"] = _tenant, + ["alias"] = _alias, + ["product"] = _product, + ["statuses"] = new BsonDocument(_statusCounts.Select(kvp => new BsonElement(kvp.Key, kvp.Value))), + ["justifications"] = new BsonArray(_justifications.Select(justification => justification)), + ["last_updated"] = _latest.HasValue ? _latest.Value : (BsonValue)BsonNull.Value, + }; + + return payload; + } + } +} + +internal sealed record VexAggregationResult( + int LinksetCount, + int StatementCount, + int EventCount, + IReadOnlyList EventDocuments); diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexObservationGenerator.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexObservationGenerator.cs new file mode 100644 index 00000000..27bd9678 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexObservationGenerator.cs @@ -0,0 +1,252 @@ +using System.Collections.Immutable; +using System.Security.Cryptography; +using MongoDB.Bson; + +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal static class VexObservationGenerator +{ + private static readonly ImmutableArray StatusPool = ImmutableArray.Create( + "affected", + "not_affected", + "under_investigation"); + + private static readonly ImmutableArray JustificationPool = ImmutableArray.Create( + "exploitation_mitigated", + "component_not_present", + "vulnerable_code_not_present", + "vulnerable_code_not_in_execute_path"); + + public static IReadOnlyList Generate(VexScenarioConfig config) + { + ArgumentNullException.ThrowIfNull(config); + + var observationCount = config.ResolveObservationCount(); + var aliasGroups = config.ResolveAliasGroups(); + var statementsPerObservation = config.ResolveStatementsPerObservation(); + var tenantCount = config.ResolveTenantCount(); + var productsPerObservation = config.ResolveProductsPerObservation(); + var seed = config.ResolveSeed(); + + var seeds = new VexObservationSeed[observationCount]; + var random = new Random(seed); + var baseTime = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero); + + for (var index = 0; index < observationCount; index++) + { + var tenantIndex = index % tenantCount; + var tenant = $"tenant-{tenantIndex:D2}"; + var group = index % aliasGroups; + var revision = index / aliasGroups; + var vulnerabilityAlias = $"CVE-2025-{group:D4}"; + var upstreamId = $"VEX-{group:D4}-{revision:D3}"; + var observationId = $"{tenant}:vex:{group:D5}:{revision:D6}"; + + var fetchedAt = baseTime.AddMinutes(revision); + var receivedAt = fetchedAt.AddSeconds(2); + var documentVersion = fetchedAt.AddSeconds(15).ToString("O"); + + var products = CreateProducts(group, revision, productsPerObservation); + var statements = CreateStatements(vulnerabilityAlias, products, statementsPerObservation, random, fetchedAt); + var rawPayload = CreateRawPayload(upstreamId, vulnerabilityAlias, statements); + var contentHash = ComputeContentHash(rawPayload, tenant, group, revision); + + var aliases = ImmutableArray.Create(vulnerabilityAlias, $"GHSA-{group:D4}-{revision % 26 + 'a'}{revision % 26 + 'a'}"); + var references = ImmutableArray.Create( + new VexReference("advisory", $"https://vendor.example/advisories/{vulnerabilityAlias.ToLowerInvariant()}"), + new VexReference("fix", $"https://vendor.example/patch/{vulnerabilityAlias.ToLowerInvariant()}")); + + seeds[index] = new VexObservationSeed( + ObservationId: observationId, + Tenant: tenant, + Vendor: "excititor-bench", + Stream: "simulated", + Api: $"https://bench.stella/vex/{group:D4}/{revision:D3}", + CollectorVersion: "1.0.0-bench", + UpstreamId: upstreamId, + DocumentVersion: documentVersion, + FetchedAt: fetchedAt, + ReceivedAt: receivedAt, + ContentHash: contentHash, + VulnerabilityAlias: vulnerabilityAlias, + Aliases: aliases, + Products: products, + Statements: statements, + References: references, + ContentFormat: "CycloneDX-VEX", + SpecVersion: "1.4", + RawPayload: rawPayload); + } + + return seeds; + } + + private static ImmutableArray CreateProducts(int group, int revision, int count) + { + var builder = ImmutableArray.CreateBuilder(count); + for (var index = 0; index < count; index++) + { + var purl = $"pkg:generic/stella/product-{group:D4}-{index}@{1 + revision % 5}.{index + 1}.{revision % 9}"; + builder.Add(new VexProduct(purl, $"component-{group % 30:D2}", $"namespace-{group % 10:D2}")); + } + + return builder.MoveToImmutable(); + } + + private static ImmutableArray CreateStatements( + string vulnerabilityAlias, + ImmutableArray products, + int statementsPerObservation, + Random random, + DateTimeOffset baseTime) + { + var builder = ImmutableArray.CreateBuilder(statementsPerObservation); + for (var index = 0; index < statementsPerObservation; index++) + { + var statusIndex = random.Next(StatusPool.Length); + var status = StatusPool[statusIndex]; + var justification = JustificationPool[random.Next(JustificationPool.Length)]; + var product = products[index % products.Length]; + var statementId = $"stmt-{vulnerabilityAlias}-{index:D2}"; + + var document = new BsonDocument + { + ["statement_id"] = statementId, + ["vulnerability_alias"] = vulnerabilityAlias, + ["product"] = new BsonDocument + { + ["purl"] = product.Purl, + ["component"] = product.Component, + ["namespace"] = product.Namespace, + }, + ["status"] = status, + ["justification"] = justification, + ["impact"] = status == "affected" ? "high" : "none", + ["last_updated"] = baseTime.AddMinutes(index).UtcDateTime, + }; + + builder.Add(document); + } + + return builder.MoveToImmutable(); + } + + private static BsonDocument CreateRawPayload(string upstreamId, string vulnerabilityAlias, ImmutableArray statements) + { + var doc = new BsonDocument + { + ["documentId"] = upstreamId, + ["title"] = $"Simulated VEX report {upstreamId}", + ["summary"] = $"Synthetic VEX payload for {vulnerabilityAlias}.", + ["statements"] = new BsonArray(statements), + }; + + return doc; + } + + private static string ComputeContentHash(BsonDocument rawPayload, string tenant, int group, int revision) + { + using var sha256 = SHA256.Create(); + var seed = $"{tenant}|{group}|{revision}"; + var rawBytes = rawPayload.ToBson(); + var seedBytes = System.Text.Encoding.UTF8.GetBytes(seed); + var combined = new byte[rawBytes.Length + seedBytes.Length]; + Buffer.BlockCopy(rawBytes, 0, combined, 0, rawBytes.Length); + Buffer.BlockCopy(seedBytes, 0, combined, rawBytes.Length, seedBytes.Length); + var hash = sha256.ComputeHash(combined); + return $"sha256:{Convert.ToHexString(hash)}"; + } +} + +internal sealed record VexObservationSeed( + string ObservationId, + string Tenant, + string Vendor, + string Stream, + string Api, + string CollectorVersion, + string UpstreamId, + string DocumentVersion, + DateTimeOffset FetchedAt, + DateTimeOffset ReceivedAt, + string ContentHash, + string VulnerabilityAlias, + ImmutableArray Aliases, + ImmutableArray Products, + ImmutableArray Statements, + ImmutableArray References, + string ContentFormat, + string SpecVersion, + BsonDocument RawPayload) +{ + public BsonDocument ToBsonDocument() + { + var aliases = new BsonArray(Aliases.Select(alias => alias)); + var statements = new BsonArray(Statements); + var productsArray = new BsonArray(Products.Select(product => new BsonDocument + { + ["purl"] = product.Purl, + ["component"] = product.Component, + ["namespace"] = product.Namespace, + })); + var references = new BsonArray(References.Select(reference => new BsonDocument + { + ["type"] = reference.Type, + ["url"] = reference.Url, + })); + + var document = new BsonDocument + { + ["_id"] = ObservationId, + ["tenant"] = Tenant, + ["source"] = new BsonDocument + { + ["vendor"] = Vendor, + ["stream"] = Stream, + ["api"] = Api, + ["collector_version"] = CollectorVersion, + }, + ["upstream"] = new BsonDocument + { + ["upstream_id"] = UpstreamId, + ["document_version"] = DocumentVersion, + ["fetched_at"] = FetchedAt.UtcDateTime, + ["received_at"] = ReceivedAt.UtcDateTime, + ["content_hash"] = ContentHash, + ["signature"] = new BsonDocument + { + ["present"] = false, + ["format"] = BsonNull.Value, + ["key_id"] = BsonNull.Value, + ["signature"] = BsonNull.Value, + }, + }, + ["content"] = new BsonDocument + { + ["format"] = ContentFormat, + ["spec_version"] = SpecVersion, + ["raw"] = RawPayload, + }, + ["identifiers"] = new BsonDocument + { + ["aliases"] = aliases, + ["primary"] = VulnerabilityAlias, + }, + ["statements"] = statements, + ["linkset"] = new BsonDocument + { + ["aliases"] = aliases, + ["products"] = productsArray, + ["references"] = references, + ["reconciled_from"] = new BsonArray { "/statements" }, + }, + ["supersedes"] = BsonNull.Value, + }; + + return document; + } +} + +internal sealed record VexProduct(string Purl, string Component, string Namespace); + +internal sealed record VexReference(string Type, string Url); diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioConfig.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioConfig.cs new file mode 100644 index 00000000..9feeb997 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioConfig.cs @@ -0,0 +1,183 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal sealed record VexBenchmarkConfig( + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MinEventThroughputPerSecond, + double? MaxAllocatedMb, + int? Iterations, + IReadOnlyList Scenarios) +{ + public static async Task LoadAsync(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + throw new FileNotFoundException($"Benchmark configuration '{resolved}' was not found.", resolved); + } + + await using var stream = File.OpenRead(resolved); + var model = await JsonSerializer.DeserializeAsync( + stream, + new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }).ConfigureAwait(false); + + if (model is null) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' could not be parsed."); + } + + if (model.Scenarios.Count == 0) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' does not contain any scenarios."); + } + + foreach (var scenario in model.Scenarios) + { + scenario.Validate(); + } + + return new VexBenchmarkConfig( + model.ThresholdMs, + model.MinThroughputPerSecond, + model.MinEventThroughputPerSecond, + model.MaxAllocatedMb, + model.Iterations, + model.Scenarios); + } + + private sealed class VexBenchmarkConfigModel + { + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("minEventThroughputPerSecond")] + public double? MinEventThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("scenarios")] + public List Scenarios { get; init; } = new(); + } +} + +internal sealed class VexScenarioConfig +{ + private const int DefaultObservationCount = 4_000; + private const int DefaultAliasGroups = 400; + private const int DefaultStatementsPerObservation = 6; + private const int DefaultProductsPerObservation = 3; + private const int DefaultTenants = 3; + private const int DefaultBatchSize = 250; + private const int DefaultSeed = 520_025; + + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } + + [JsonPropertyName("observations")] + public int? Observations { get; init; } + + [JsonPropertyName("aliasGroups")] + public int? AliasGroups { get; init; } + + [JsonPropertyName("statementsPerObservation")] + public int? StatementsPerObservation { get; init; } + + [JsonPropertyName("productsPerObservation")] + public int? ProductsPerObservation { get; init; } + + [JsonPropertyName("tenants")] + public int? Tenants { get; init; } + + [JsonPropertyName("batchSize")] + public int? BatchSize { get; init; } + + [JsonPropertyName("seed")] + public int? Seed { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("minEventThroughputPerSecond")] + public double? MinEventThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + public string ScenarioId => string.IsNullOrWhiteSpace(Id) ? "vex" : Id!.Trim(); + + public string DisplayLabel => string.IsNullOrWhiteSpace(Label) ? ScenarioId : Label!.Trim(); + + public int ResolveObservationCount() => Observations is > 0 ? Observations.Value : DefaultObservationCount; + + public int ResolveAliasGroups() => AliasGroups is > 0 ? AliasGroups.Value : DefaultAliasGroups; + + public int ResolveStatementsPerObservation() => StatementsPerObservation is > 0 ? StatementsPerObservation.Value : DefaultStatementsPerObservation; + + public int ResolveProductsPerObservation() => ProductsPerObservation is > 0 ? ProductsPerObservation.Value : DefaultProductsPerObservation; + + public int ResolveTenantCount() => Tenants is > 0 ? Tenants.Value : DefaultTenants; + + public int ResolveBatchSize() => BatchSize is > 0 ? BatchSize.Value : DefaultBatchSize; + + public int ResolveSeed() => Seed is > 0 ? Seed.Value : DefaultSeed; + + public int ResolveIterations(int? defaultIterations) + { + var iterations = Iterations ?? defaultIterations ?? 3; + if (iterations <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires iterations > 0."); + } + + return iterations; + } + + public void Validate() + { + if (ResolveObservationCount() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires observations > 0."); + } + + if (ResolveAliasGroups() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires aliasGroups > 0."); + } + + if (ResolveStatementsPerObservation() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires statementsPerObservation > 0."); + } + + if (ResolveProductsPerObservation() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires productsPerObservation > 0."); + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioExecutionResult.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioExecutionResult.cs new file mode 100644 index 00000000..f0cd962b --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioExecutionResult.cs @@ -0,0 +1,14 @@ +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal sealed record VexScenarioExecutionResult( + IReadOnlyList TotalDurationsMs, + IReadOnlyList InsertDurationsMs, + IReadOnlyList CorrelationDurationsMs, + IReadOnlyList AllocatedMb, + IReadOnlyList ObservationThroughputsPerSecond, + IReadOnlyList EventThroughputsPerSecond, + int ObservationCount, + int AliasGroups, + int StatementCount, + int EventCount, + VexAggregationResult AggregationResult); diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioResult.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioResult.cs new file mode 100644 index 00000000..f69e3143 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioResult.cs @@ -0,0 +1,43 @@ +using System.Globalization; + +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal sealed record VexScenarioResult( + string Id, + string Label, + int Iterations, + int ObservationCount, + int AliasGroups, + int StatementCount, + int EventCount, + DurationStatistics TotalStatistics, + DurationStatistics InsertStatistics, + DurationStatistics CorrelationStatistics, + ThroughputStatistics ObservationThroughputStatistics, + ThroughputStatistics EventThroughputStatistics, + AllocationStatistics AllocationStatistics, + double? ThresholdMs, + double? MinObservationThroughputPerSecond, + double? MinEventThroughputPerSecond, + double? MaxAllocatedThresholdMb) +{ + public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28]; + + public string ObservationsColumn => ObservationCount.ToString("N0", CultureInfo.InvariantCulture).PadLeft(12); + + public string StatementColumn => StatementCount.ToString("N0", CultureInfo.InvariantCulture).PadLeft(10); + + public string EventColumn => EventCount.ToString("N0", CultureInfo.InvariantCulture).PadLeft(8); + + public string TotalMeanColumn => TotalStatistics.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string CorrelationMeanColumn => CorrelationStatistics.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string InsertMeanColumn => InsertStatistics.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string ObservationThroughputColumn => (ObservationThroughputStatistics.MinPerSecond / 1_000d).ToString("F2", CultureInfo.InvariantCulture).PadLeft(11); + + public string EventThroughputColumn => (EventThroughputStatistics.MinPerSecond / 1_000d).ToString("F2", CultureInfo.InvariantCulture).PadLeft(11); + + public string AllocatedColumn => AllocationStatistics.MaxAllocatedMb.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9); +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioRunner.cs b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioRunner.cs new file mode 100644 index 00000000..50bf21f2 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioRunner.cs @@ -0,0 +1,138 @@ +using System.Diagnostics; +using EphemeralMongo; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Bench.LinkNotMerge.Vex; + +internal sealed class VexScenarioRunner +{ + private readonly VexScenarioConfig _config; + private readonly IReadOnlyList _seeds; + + public VexScenarioRunner(VexScenarioConfig config) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _seeds = VexObservationGenerator.Generate(config); + } + + public VexScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken) + { + if (iterations <= 0) + { + throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); + } + + var totalDurations = new double[iterations]; + var insertDurations = new double[iterations]; + var correlationDurations = new double[iterations]; + var allocated = new double[iterations]; + var observationThroughputs = new double[iterations]; + var eventThroughputs = new double[iterations]; + VexAggregationResult lastAggregation = new(0, 0, 0, Array.Empty()); + + for (var iteration = 0; iteration < iterations; iteration++) + { + cancellationToken.ThrowIfCancellationRequested(); + + using var runner = MongoRunner.Run(new MongoRunnerOptions + { + UseSingleNodeReplicaSet = false, + }); + + var client = new MongoClient(runner.ConnectionString); + var database = client.GetDatabase("linknotmerge_vex_bench"); + var collection = database.GetCollection("vex_observations"); + + CreateIndexes(collection, cancellationToken); + + var beforeAllocated = GC.GetTotalAllocatedBytes(); + + var insertStopwatch = Stopwatch.StartNew(); + InsertObservations(collection, _seeds, _config.ResolveBatchSize(), cancellationToken); + insertStopwatch.Stop(); + + var correlationStopwatch = Stopwatch.StartNew(); + var documents = collection + .Find(FilterDefinition.Empty) + .Project(Builders.Projection + .Include("tenant") + .Include("statements") + .Include("linkset")) + .ToList(cancellationToken); + + var aggregator = new VexLinksetAggregator(); + lastAggregation = aggregator.Correlate(documents); + correlationStopwatch.Stop(); + + var totalElapsed = insertStopwatch.Elapsed + correlationStopwatch.Elapsed; + var afterAllocated = GC.GetTotalAllocatedBytes(); + + totalDurations[iteration] = totalElapsed.TotalMilliseconds; + insertDurations[iteration] = insertStopwatch.Elapsed.TotalMilliseconds; + correlationDurations[iteration] = correlationStopwatch.Elapsed.TotalMilliseconds; + allocated[iteration] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d); + + var totalSeconds = Math.Max(totalElapsed.TotalSeconds, 0.0001d); + observationThroughputs[iteration] = _seeds.Count / totalSeconds; + + var eventSeconds = Math.Max(correlationStopwatch.Elapsed.TotalSeconds, 0.0001d); + var eventCount = Math.Max(lastAggregation.EventCount, 1); + eventThroughputs[iteration] = eventCount / eventSeconds; + } + + return new VexScenarioExecutionResult( + totalDurations, + insertDurations, + correlationDurations, + allocated, + observationThroughputs, + eventThroughputs, + ObservationCount: _seeds.Count, + AliasGroups: _config.ResolveAliasGroups(), + StatementCount: lastAggregation.StatementCount, + EventCount: lastAggregation.EventCount, + AggregationResult: lastAggregation); + } + + private static void InsertObservations( + IMongoCollection collection, + IReadOnlyList seeds, + int batchSize, + CancellationToken cancellationToken) + { + for (var offset = 0; offset < seeds.Count; offset += batchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + + var remaining = Math.Min(batchSize, seeds.Count - offset); + var batch = new List(remaining); + for (var index = 0; index < remaining; index++) + { + batch.Add(seeds[offset + index].ToBsonDocument()); + } + + collection.InsertMany(batch, new InsertManyOptions + { + IsOrdered = false, + BypassDocumentValidation = true, + }, cancellationToken); + } + } + + private static void CreateIndexes(IMongoCollection collection, CancellationToken cancellationToken) + { + var indexKeys = Builders.IndexKeys + .Ascending("tenant") + .Ascending("linkset.aliases"); + + try + { + collection.Indexes.CreateOne(new CreateIndexModel(indexKeys), cancellationToken: cancellationToken); + } + catch + { + // non-fatal + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/baseline.csv b/src/StellaOps.Bench/LinkNotMerge.Vex/baseline.csv new file mode 100644 index 00000000..34578590 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/baseline.csv @@ -0,0 +1,4 @@ +scenario,iterations,observations,statements,events,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_observation_throughput_per_sec,min_observation_throughput_per_sec,mean_event_throughput_per_sec,min_event_throughput_per_sec,max_allocated_mb +vex_ingest_baseline,5,4000,24000,21326,842.8191,1319.3038,1432.7675,346.7277,496.0915,5349.8940,2791.7998,48942.4901,24653.0556,138.6365 +vex_ingest_medium,5,8000,64000,56720,1525.9929,1706.8900,1748.9056,533.3378,992.6552,5274.5883,4574.2892,57654.9190,48531.7353,326.8638 +vex_ingest_high,5,12000,120000,106910,2988.5094,3422.1728,3438.9364,903.3927,2085.1167,4066.2300,3489.4510,52456.9493,42358.0556,583.9903 diff --git a/src/StellaOps.Bench/LinkNotMerge.Vex/config.json b/src/StellaOps.Bench/LinkNotMerge.Vex/config.json new file mode 100644 index 00000000..e20cbd7f --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge.Vex/config.json @@ -0,0 +1,54 @@ +{ + "thresholdMs": 4200, + "minThroughputPerSecond": 1800, + "minEventThroughputPerSecond": 2000, + "maxAllocatedMb": 800, + "iterations": 5, + "scenarios": [ + { + "id": "vex_ingest_baseline", + "label": "4k observations, 400 aliases", + "observations": 4000, + "aliasGroups": 400, + "statementsPerObservation": 6, + "productsPerObservation": 3, + "tenants": 3, + "batchSize": 200, + "seed": 420020, + "thresholdMs": 2300, + "minThroughputPerSecond": 1800, + "minEventThroughputPerSecond": 2000, + "maxAllocatedMb": 220 + }, + { + "id": "vex_ingest_medium", + "label": "8k observations, 700 aliases", + "observations": 8000, + "aliasGroups": 700, + "statementsPerObservation": 8, + "productsPerObservation": 4, + "tenants": 5, + "batchSize": 300, + "seed": 520020, + "thresholdMs": 3200, + "minThroughputPerSecond": 2200, + "minEventThroughputPerSecond": 2500, + "maxAllocatedMb": 400 + }, + { + "id": "vex_ingest_high", + "label": "12k observations, 1100 aliases", + "observations": 12000, + "aliasGroups": 1100, + "statementsPerObservation": 10, + "productsPerObservation": 5, + "tenants": 7, + "batchSize": 400, + "seed": 620020, + "thresholdMs": 4200, + "minThroughputPerSecond": 2200, + "minEventThroughputPerSecond": 2500, + "maxAllocatedMb": 700 + } + ] +} diff --git a/src/StellaOps.Bench/LinkNotMerge/README.md b/src/StellaOps.Bench/LinkNotMerge/README.md new file mode 100644 index 00000000..e2e5dfc5 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/README.md @@ -0,0 +1,26 @@ +# Link-Not-Merge Bench + +Synthetic workload that measures advisory observation ingestion and linkset correlation throughput for the Link-Not-Merge program. + +## Scenarios + +`config.json` defines three scenarios that vary observation volume, alias density, and correlation fan-out. Each scenario captures: + +- Total latency (ingest + correlation) and p95/max percentiles +- Insert latency against an ephemeral MongoDB instance +- Correlator-only latency, tracking fan-out costs +- Observation and Mongo insert throughput (ops/sec) +- Peak managed heap allocations + +## Running locally + +```bash +dotnet run \ + --project src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/StellaOps.Bench.LinkNotMerge.csproj \ + -- \ + --csv out/linknotmerge-bench.csv \ + --json out/linknotmerge-bench.json \ + --prometheus out/linknotmerge-bench.prom +``` + +The benchmark exits non-zero if latency exceeds configured thresholds, throughput falls below the floor, Mongo insert throughput regresses, allocations exceed the ceiling, or regression ratios breach the baseline. diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/BaselineLoaderTests.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/BaselineLoaderTests.cs new file mode 100644 index 00000000..eec70e2b --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/BaselineLoaderTests.cs @@ -0,0 +1,38 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Bench.LinkNotMerge.Baseline; +using Xunit; + +namespace StellaOps.Bench.LinkNotMerge.Tests; + +public sealed class BaselineLoaderTests +{ + [Fact] + public async Task LoadAsync_ReadsEntries() + { + var path = Path.GetTempFileName(); + try + { + await File.WriteAllTextAsync( + path, + "scenario,iterations,observations,aliases,linksets,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_throughput_per_sec,min_throughput_per_sec,mean_mongo_throughput_per_sec,min_mongo_throughput_per_sec,max_allocated_mb\n" + + "lnm_ingest_baseline,5,5000,500,450,320.5,340.1,360.9,120.2,210.3,15000.0,13500.0,18000.0,16500.0,96.5\n"); + + var baseline = await BaselineLoader.LoadAsync(path, CancellationToken.None); + var entry = Assert.Single(baseline); + + Assert.Equal("lnm_ingest_baseline", entry.Key); + Assert.Equal(5, entry.Value.Iterations); + Assert.Equal(5000, entry.Value.Observations); + Assert.Equal(500, entry.Value.Aliases); + Assert.Equal(360.9, entry.Value.MaxTotalMs); + Assert.Equal(16500.0, entry.Value.MinMongoThroughputPerSecond); + Assert.Equal(96.5, entry.Value.MaxAllocatedMb); + } + finally + { + File.Delete(path); + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/BenchmarkScenarioReportTests.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/BenchmarkScenarioReportTests.cs new file mode 100644 index 00000000..3f3312fa --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/BenchmarkScenarioReportTests.cs @@ -0,0 +1,81 @@ +using StellaOps.Bench.LinkNotMerge.Baseline; +using StellaOps.Bench.LinkNotMerge.Reporting; +using Xunit; + +namespace StellaOps.Bench.LinkNotMerge.Tests; + +public sealed class BenchmarkScenarioReportTests +{ + [Fact] + public void RegressionDetection_FlagsBreaches() + { + var result = new ScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + ObservationCount: 1000, + AliasGroups: 100, + LinksetCount: 90, + TotalStatistics: new DurationStatistics(200, 240, 260), + InsertStatistics: new DurationStatistics(80, 90, 100), + CorrelationStatistics: new DurationStatistics(120, 150, 170), + TotalThroughputStatistics: new ThroughputStatistics(8000, 7000), + InsertThroughputStatistics: new ThroughputStatistics(9000, 8000), + AllocationStatistics: new AllocationStatistics(120), + ThresholdMs: null, + MinThroughputThresholdPerSecond: null, + MinMongoThroughputThresholdPerSecond: null, + MaxAllocatedThresholdMb: null); + + var baseline = new BaselineEntry( + ScenarioId: "scenario", + Iterations: 3, + Observations: 1000, + Aliases: 100, + Linksets: 90, + MeanTotalMs: 150, + P95TotalMs: 170, + MaxTotalMs: 180, + MeanInsertMs: 60, + MeanCorrelationMs: 90, + MeanThroughputPerSecond: 9000, + MinThroughputPerSecond: 8500, + MeanMongoThroughputPerSecond: 10000, + MinMongoThroughputPerSecond: 9500, + MaxAllocatedMb: 100); + + var report = new BenchmarkScenarioReport(result, baseline, regressionLimit: 1.1); + + Assert.True(report.DurationRegressionBreached); + Assert.True(report.ThroughputRegressionBreached); + Assert.True(report.MongoThroughputRegressionBreached); + Assert.Contains(report.BuildRegressionFailureMessages(), message => message.Contains("max duration")); + } + + [Fact] + public void RegressionDetection_NoBaseline_NoBreaches() + { + var result = new ScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + ObservationCount: 1000, + AliasGroups: 100, + LinksetCount: 90, + TotalStatistics: new DurationStatistics(200, 220, 230), + InsertStatistics: new DurationStatistics(90, 100, 110), + CorrelationStatistics: new DurationStatistics(110, 120, 130), + TotalThroughputStatistics: new ThroughputStatistics(8000, 7900), + InsertThroughputStatistics: new ThroughputStatistics(9000, 8900), + AllocationStatistics: new AllocationStatistics(64), + ThresholdMs: null, + MinThroughputThresholdPerSecond: null, + MinMongoThroughputThresholdPerSecond: null, + MaxAllocatedThresholdMb: null); + + var report = new BenchmarkScenarioReport(result, baseline: null, regressionLimit: null); + + Assert.False(report.RegressionBreached); + Assert.Empty(report.BuildRegressionFailureMessages()); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/LinkNotMergeScenarioRunnerTests.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/LinkNotMergeScenarioRunnerTests.cs new file mode 100644 index 00000000..e98fcb15 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/LinkNotMergeScenarioRunnerTests.cs @@ -0,0 +1,38 @@ +using System.Linq; +using System.Threading; +using StellaOps.Bench.LinkNotMerge.Baseline; +using Xunit; + +namespace StellaOps.Bench.LinkNotMerge.Tests; + +public sealed class LinkNotMergeScenarioRunnerTests +{ + [Fact] + public void Execute_BuildsDeterministicAggregation() + { + var config = new LinkNotMergeScenarioConfig + { + Id = "unit", + Observations = 120, + AliasGroups = 24, + PurlsPerObservation = 3, + CpesPerObservation = 2, + ReferencesPerObservation = 2, + Tenants = 3, + BatchSize = 40, + Seed = 1337, + }; + + var runner = new LinkNotMergeScenarioRunner(config); + var result = runner.Execute(iterations: 2, CancellationToken.None); + + Assert.Equal(120, result.ObservationCount); + Assert.Equal(24, result.AliasGroups); + Assert.True(result.TotalDurationsMs.All(value => value > 0)); + Assert.True(result.InsertThroughputsPerSecond.All(value => value > 0)); + Assert.True(result.TotalThroughputsPerSecond.All(value => value > 0)); + Assert.True(result.AllocatedMb.All(value => value >= 0)); + Assert.Equal(result.AggregationResult.LinksetCount, result.LinksetCount); + Assert.Equal(result.AggregationResult.ObservationCount, result.ObservationCount); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/StellaOps.Bench.LinkNotMerge.Tests.csproj b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/StellaOps.Bench.LinkNotMerge.Tests.csproj new file mode 100644 index 00000000..3a6c039d --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/StellaOps.Bench.LinkNotMerge.Tests.csproj @@ -0,0 +1,28 @@ + + + net10.0 + enable + enable + preview + true + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Baseline/BaselineEntry.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Baseline/BaselineEntry.cs new file mode 100644 index 00000000..a5b3bfc2 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Baseline/BaselineEntry.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Bench.LinkNotMerge.Baseline; + +internal sealed record BaselineEntry( + string ScenarioId, + int Iterations, + int Observations, + int Aliases, + int Linksets, + double MeanTotalMs, + double P95TotalMs, + double MaxTotalMs, + double MeanInsertMs, + double MeanCorrelationMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MeanMongoThroughputPerSecond, + double MinMongoThroughputPerSecond, + double MaxAllocatedMb); diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Baseline/BaselineLoader.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Baseline/BaselineLoader.cs new file mode 100644 index 00000000..c7f67b68 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Baseline/BaselineLoader.cs @@ -0,0 +1,87 @@ +using System.Globalization; + +namespace StellaOps.Bench.LinkNotMerge.Baseline; + +internal static class BaselineLoader +{ + public static async Task> LoadAsync(string path, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read); + using var reader = new StreamReader(stream); + + var lineNumber = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var line = await reader.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + lineNumber++; + if (lineNumber == 1 || string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = line.Split(',', StringSplitOptions.TrimEntries); + if (parts.Length < 15) + { + throw new InvalidOperationException($"Baseline '{resolved}' line {lineNumber} is invalid (expected 15 columns, found {parts.Length})."); + } + + var entry = new BaselineEntry( + ScenarioId: parts[0], + Iterations: ParseInt(parts[1], resolved, lineNumber), + Observations: ParseInt(parts[2], resolved, lineNumber), + Aliases: ParseInt(parts[3], resolved, lineNumber), + Linksets: ParseInt(parts[4], resolved, lineNumber), + MeanTotalMs: ParseDouble(parts[5], resolved, lineNumber), + P95TotalMs: ParseDouble(parts[6], resolved, lineNumber), + MaxTotalMs: ParseDouble(parts[7], resolved, lineNumber), + MeanInsertMs: ParseDouble(parts[8], resolved, lineNumber), + MeanCorrelationMs: ParseDouble(parts[9], resolved, lineNumber), + MeanThroughputPerSecond: ParseDouble(parts[10], resolved, lineNumber), + MinThroughputPerSecond: ParseDouble(parts[11], resolved, lineNumber), + MeanMongoThroughputPerSecond: ParseDouble(parts[12], resolved, lineNumber), + MinMongoThroughputPerSecond: ParseDouble(parts[13], resolved, lineNumber), + MaxAllocatedMb: ParseDouble(parts[14], resolved, lineNumber)); + + result[entry.ScenarioId] = entry; + } + + return result; + } + + private static int ParseInt(string value, string file, int line) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var result)) + { + return result; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid integer '{value}'."); + } + + private static double ParseDouble(string value, string file, int line) + { + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var result)) + { + return result; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid number '{value}'."); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/BenchmarkConfig.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/BenchmarkConfig.cs new file mode 100644 index 00000000..2aebc423 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/BenchmarkConfig.cs @@ -0,0 +1,210 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.LinkNotMerge; + +internal sealed record BenchmarkConfig( + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MinMongoThroughputPerSecond, + double? MaxAllocatedMb, + int? Iterations, + IReadOnlyList Scenarios) +{ + public static async Task LoadAsync(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + throw new FileNotFoundException($"Benchmark configuration '{resolved}' was not found.", resolved); + } + + await using var stream = File.OpenRead(resolved); + var model = await JsonSerializer.DeserializeAsync( + stream, + new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }).ConfigureAwait(false); + + if (model is null) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' could not be parsed."); + } + + if (model.Scenarios.Count == 0) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' does not contain any scenarios."); + } + + foreach (var scenario in model.Scenarios) + { + scenario.Validate(); + } + + return new BenchmarkConfig( + model.ThresholdMs, + model.MinThroughputPerSecond, + model.MinMongoThroughputPerSecond, + model.MaxAllocatedMb, + model.Iterations, + model.Scenarios); + } + + private sealed class BenchmarkConfigModel + { + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("minMongoThroughputPerSecond")] + public double? MinMongoThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("scenarios")] + public List Scenarios { get; init; } = new(); + } +} + +internal sealed class LinkNotMergeScenarioConfig +{ + private const int DefaultObservationCount = 5_000; + private const int DefaultAliasGroups = 500; + private const int DefaultPurlsPerObservation = 4; + private const int DefaultCpesPerObservation = 2; + private const int DefaultReferencesPerObservation = 3; + private const int DefaultTenants = 4; + private const int DefaultBatchSize = 500; + private const int DefaultSeed = 42_022; + + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } + + [JsonPropertyName("observations")] + public int? Observations { get; init; } + + [JsonPropertyName("aliasGroups")] + public int? AliasGroups { get; init; } + + [JsonPropertyName("purlsPerObservation")] + public int? PurlsPerObservation { get; init; } + + [JsonPropertyName("cpesPerObservation")] + public int? CpesPerObservation { get; init; } + + [JsonPropertyName("referencesPerObservation")] + public int? ReferencesPerObservation { get; init; } + + [JsonPropertyName("tenants")] + public int? Tenants { get; init; } + + [JsonPropertyName("batchSize")] + public int? BatchSize { get; init; } + + [JsonPropertyName("seed")] + public int? Seed { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("minMongoThroughputPerSecond")] + public double? MinMongoThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + public string ScenarioId => string.IsNullOrWhiteSpace(Id) ? "linknotmerge" : Id!.Trim(); + + public string DisplayLabel => string.IsNullOrWhiteSpace(Label) ? ScenarioId : Label!.Trim(); + + public int ResolveObservationCount() => Observations.HasValue && Observations.Value > 0 + ? Observations.Value + : DefaultObservationCount; + + public int ResolveAliasGroups() => AliasGroups.HasValue && AliasGroups.Value > 0 + ? AliasGroups.Value + : DefaultAliasGroups; + + public int ResolvePurlsPerObservation() => PurlsPerObservation.HasValue && PurlsPerObservation.Value > 0 + ? PurlsPerObservation.Value + : DefaultPurlsPerObservation; + + public int ResolveCpesPerObservation() => CpesPerObservation.HasValue && CpesPerObservation.Value >= 0 + ? CpesPerObservation.Value + : DefaultCpesPerObservation; + + public int ResolveReferencesPerObservation() => ReferencesPerObservation.HasValue && ReferencesPerObservation.Value >= 0 + ? ReferencesPerObservation.Value + : DefaultReferencesPerObservation; + + public int ResolveTenantCount() => Tenants.HasValue && Tenants.Value > 0 + ? Tenants.Value + : DefaultTenants; + + public int ResolveBatchSize() => BatchSize.HasValue && BatchSize.Value > 0 + ? BatchSize.Value + : DefaultBatchSize; + + public int ResolveSeed() => Seed.HasValue && Seed.Value > 0 + ? Seed.Value + : DefaultSeed; + + public int ResolveIterations(int? defaultIterations) + { + var iterations = Iterations ?? defaultIterations ?? 3; + if (iterations <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires iterations > 0."); + } + + return iterations; + } + + public void Validate() + { + if (ResolveObservationCount() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires observations > 0."); + } + + if (ResolveAliasGroups() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires aliasGroups > 0."); + } + + if (ResolvePurlsPerObservation() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires purlsPerObservation > 0."); + } + + if (ResolveTenantCount() <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires tenants > 0."); + } + + if (ResolveBatchSize() > ResolveObservationCount()) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' batchSize cannot exceed observations."); + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinkNotMergeScenarioRunner.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinkNotMergeScenarioRunner.cs new file mode 100644 index 00000000..93a48cdd --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinkNotMergeScenarioRunner.cs @@ -0,0 +1,135 @@ +using System.Diagnostics; +using EphemeralMongo; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Bench.LinkNotMerge; + +internal sealed class LinkNotMergeScenarioRunner +{ + private readonly LinkNotMergeScenarioConfig _config; + private readonly IReadOnlyList _seeds; + + public LinkNotMergeScenarioRunner(LinkNotMergeScenarioConfig config) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _seeds = ObservationGenerator.Generate(config); + } + + public ScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken) + { + if (iterations <= 0) + { + throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); + } + + var totalDurations = new double[iterations]; + var insertDurations = new double[iterations]; + var correlationDurations = new double[iterations]; + var allocated = new double[iterations]; + var totalThroughputs = new double[iterations]; + var insertThroughputs = new double[iterations]; + LinksetAggregationResult lastAggregation = new(0, 0, 0, 0, 0); + + for (var iteration = 0; iteration < iterations; iteration++) + { + cancellationToken.ThrowIfCancellationRequested(); + + using var runner = MongoRunner.Run(new MongoRunnerOptions + { + UseSingleNodeReplicaSet = false, + }); + + var client = new MongoClient(runner.ConnectionString); + var database = client.GetDatabase("linknotmerge_bench"); + var collection = database.GetCollection("advisory_observations"); + + CreateIndexes(collection, cancellationToken); + + var beforeAllocated = GC.GetTotalAllocatedBytes(); + var insertStopwatch = Stopwatch.StartNew(); + InsertObservations(collection, _seeds, _config.ResolveBatchSize(), cancellationToken); + insertStopwatch.Stop(); + + var correlationStopwatch = Stopwatch.StartNew(); + var documents = collection + .Find(FilterDefinition.Empty) + .Project(Builders.Projection + .Include("tenant") + .Include("linkset")) + .ToList(cancellationToken); + + var correlator = new LinksetAggregator(); + lastAggregation = correlator.Correlate(documents); + correlationStopwatch.Stop(); + + var totalElapsed = insertStopwatch.Elapsed + correlationStopwatch.Elapsed; + var afterAllocated = GC.GetTotalAllocatedBytes(); + + totalDurations[iteration] = totalElapsed.TotalMilliseconds; + insertDurations[iteration] = insertStopwatch.Elapsed.TotalMilliseconds; + correlationDurations[iteration] = correlationStopwatch.Elapsed.TotalMilliseconds; + allocated[iteration] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d); + + var totalSeconds = Math.Max(totalElapsed.TotalSeconds, 0.0001d); + totalThroughputs[iteration] = _seeds.Count / totalSeconds; + + var insertSeconds = Math.Max(insertStopwatch.Elapsed.TotalSeconds, 0.0001d); + insertThroughputs[iteration] = _seeds.Count / insertSeconds; + } + + return new ScenarioExecutionResult( + totalDurations, + insertDurations, + correlationDurations, + allocated, + totalThroughputs, + insertThroughputs, + ObservationCount: _seeds.Count, + AliasGroups: _config.ResolveAliasGroups(), + LinksetCount: lastAggregation.LinksetCount, + TenantCount: _config.ResolveTenantCount(), + AggregationResult: lastAggregation); + } + + private static void InsertObservations( + IMongoCollection collection, + IReadOnlyList seeds, + int batchSize, + CancellationToken cancellationToken) + { + for (var offset = 0; offset < seeds.Count; offset += batchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + + var remaining = Math.Min(batchSize, seeds.Count - offset); + var batch = new List(remaining); + for (var index = 0; index < remaining; index++) + { + batch.Add(seeds[offset + index].ToBsonDocument()); + } + + collection.InsertMany(batch, new InsertManyOptions + { + IsOrdered = false, + BypassDocumentValidation = true, + }, cancellationToken); + } + } + + private static void CreateIndexes(IMongoCollection collection, CancellationToken cancellationToken) + { + var indexKeys = Builders.IndexKeys + .Ascending("tenant") + .Ascending("identifiers.aliases"); + + try + { + collection.Indexes.CreateOne(new CreateIndexModel(indexKeys), cancellationToken: cancellationToken); + } + catch + { + // Index creation failures should not abort the benchmark; they may occur when running multiple iterations concurrently. + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinksetAggregator.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinksetAggregator.cs new file mode 100644 index 00000000..1c1910ce --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinksetAggregator.cs @@ -0,0 +1,140 @@ +using MongoDB.Bson; + +namespace StellaOps.Bench.LinkNotMerge; + +internal sealed class LinksetAggregator +{ + public LinksetAggregationResult Correlate(IEnumerable documents) + { + ArgumentNullException.ThrowIfNull(documents); + + var groups = new Dictionary(StringComparer.Ordinal); + var totalObservations = 0; + + foreach (var document in documents) + { + totalObservations++; + + var tenant = document.GetValue("tenant", "unknown").AsString; + var linkset = document.GetValue("linkset", new BsonDocument()).AsBsonDocument; + var aliases = linkset.GetValue("aliases", new BsonArray()).AsBsonArray; + var purls = linkset.GetValue("purls", new BsonArray()).AsBsonArray; + var cpes = linkset.GetValue("cpes", new BsonArray()).AsBsonArray; + var references = linkset.GetValue("references", new BsonArray()).AsBsonArray; + + foreach (var aliasValue in aliases) + { + if (!aliasValue.IsString) + { + continue; + } + + var alias = aliasValue.AsString; + var key = string.Create(alias.Length + tenant.Length + 1, (tenant, alias), static (span, data) => + { + var (tenantValue, aliasValue) = data; + tenantValue.AsSpan().CopyTo(span); + span[tenantValue.Length] = '|'; + aliasValue.AsSpan().CopyTo(span[(tenantValue.Length + 1)..]); + }); + + if (!groups.TryGetValue(key, out var accumulator)) + { + accumulator = new LinksetAccumulator(tenant, alias); + groups[key] = accumulator; + } + + accumulator.AddPurls(purls); + accumulator.AddCpes(cpes); + accumulator.AddReferences(references); + } + } + + var totalReferences = 0; + var totalPurls = 0; + var totalCpes = 0; + + foreach (var accumulator in groups.Values) + { + totalReferences += accumulator.ReferenceCount; + totalPurls += accumulator.PurlCount; + totalCpes += accumulator.CpeCount; + } + + return new LinksetAggregationResult( + LinksetCount: groups.Count, + ObservationCount: totalObservations, + TotalPurls: totalPurls, + TotalCpes: totalCpes, + TotalReferences: totalReferences); + } + + private sealed class LinksetAccumulator + { + private readonly HashSet _purls = new(StringComparer.Ordinal); + private readonly HashSet _cpes = new(StringComparer.Ordinal); + private readonly HashSet _references = new(StringComparer.Ordinal); + + public LinksetAccumulator(string tenant, string alias) + { + Tenant = tenant; + Alias = alias; + } + + public string Tenant { get; } + + public string Alias { get; } + + public int PurlCount => _purls.Count; + + public int CpeCount => _cpes.Count; + + public int ReferenceCount => _references.Count; + + public void AddPurls(BsonArray array) + { + foreach (var item in array) + { + if (item.IsString) + { + _purls.Add(item.AsString); + } + } + } + + public void AddCpes(BsonArray array) + { + foreach (var item in array) + { + if (item.IsString) + { + _cpes.Add(item.AsString); + } + } + } + + public void AddReferences(BsonArray array) + { + foreach (var item in array) + { + if (!item.IsBsonDocument) + { + continue; + } + + var document = item.AsBsonDocument; + if (document.TryGetValue("url", out var urlValue) && urlValue.IsString) + { + _references.Add(urlValue.AsString); + } + } + } + } +} + +internal sealed record LinksetAggregationResult( + int LinksetCount, + int ObservationCount, + int TotalPurls, + int TotalCpes, + int TotalReferences); diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ObservationData.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ObservationData.cs new file mode 100644 index 00000000..8f2e544b --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ObservationData.cs @@ -0,0 +1,270 @@ +using System.Collections.Immutable; +using System.Security.Cryptography; +using MongoDB.Bson; + +namespace StellaOps.Bench.LinkNotMerge; + +internal static class ObservationGenerator +{ + public static IReadOnlyList Generate(LinkNotMergeScenarioConfig config) + { + ArgumentNullException.ThrowIfNull(config); + + var observationCount = config.ResolveObservationCount(); + var aliasGroups = config.ResolveAliasGroups(); + var purlsPerObservation = config.ResolvePurlsPerObservation(); + var cpesPerObservation = config.ResolveCpesPerObservation(); + var referencesPerObservation = config.ResolveReferencesPerObservation(); + var tenantCount = config.ResolveTenantCount(); + var seed = config.ResolveSeed(); + + var seeds = new ObservationSeed[observationCount]; + var random = new Random(seed); + var baseTime = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero); + + for (var index = 0; index < observationCount; index++) + { + var tenantIndex = index % tenantCount; + var tenant = $"tenant-{tenantIndex:D2}"; + var group = index % aliasGroups; + var revision = index / aliasGroups; + var primaryAlias = $"CVE-2025-{group:D4}"; + var vendorAlias = $"VENDOR-{group:D4}"; + var thirdAlias = $"GHSA-{group:D4}-{(revision % 26 + 'a')}{(revision % 26 + 'a')}"; + var aliases = ImmutableArray.Create(primaryAlias, vendorAlias, thirdAlias); + + var observationId = $"{tenant}:advisory:{group:D5}:{revision:D6}"; + var upstreamId = primaryAlias; + var documentVersion = baseTime.AddMinutes(revision).ToString("O"); + var fetchedAt = baseTime.AddSeconds(index % 1_800); + var receivedAt = fetchedAt.AddSeconds(1); + + var purls = CreatePurls(group, revision, purlsPerObservation); + var cpes = CreateCpes(group, revision, cpesPerObservation); + var references = CreateReferences(primaryAlias, referencesPerObservation); + + var rawPayload = CreateRawPayload(primaryAlias, vendorAlias, purls, cpes, references); + var contentHash = ComputeContentHash(rawPayload, tenant, group, revision); + + seeds[index] = new ObservationSeed( + ObservationId: observationId, + Tenant: tenant, + Vendor: "concelier-bench", + Stream: "simulated", + Api: $"https://bench.stella/{group:D4}/{revision:D2}", + CollectorVersion: "1.0.0-bench", + UpstreamId: upstreamId, + DocumentVersion: documentVersion, + FetchedAt: fetchedAt, + ReceivedAt: receivedAt, + ContentHash: contentHash, + Aliases: aliases, + Purls: purls, + Cpes: cpes, + References: references, + ContentFormat: "CSAF", + SpecVersion: "2.0", + RawPayload: rawPayload); + } + + return seeds; + } + + private static ImmutableArray CreatePurls(int group, int revision, int count) + { + if (count <= 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(count); + for (var index = 0; index < count; index++) + { + var version = $"{revision % 9 + 1}.{index + 1}.{group % 10}"; + builder.Add($"pkg:generic/stella/sample-{group:D4}-{index}@{version}"); + } + + return builder.MoveToImmutable(); + } + + private static ImmutableArray CreateCpes(int group, int revision, int count) + { + if (count <= 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(count); + for (var index = 0; index < count; index++) + { + var component = $"benchtool{group % 50:D2}"; + var version = $"{revision % 5}.{index}"; + builder.Add($"cpe:2.3:a:stellaops:{component}:{version}:*:*:*:*:*:*:*"); + } + + return builder.MoveToImmutable(); + } + + private static ImmutableArray CreateReferences(string primaryAlias, int count) + { + if (count <= 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(count); + for (var index = 0; index < count; index++) + { + builder.Add(new ObservationReference( + Type: index % 2 == 0 ? "advisory" : "patch", + Url: $"https://vendor.example/{primaryAlias.ToLowerInvariant()}/ref/{index:D2}")); + } + + return builder.MoveToImmutable(); + } + + private static BsonDocument CreateRawPayload( + string primaryAlias, + string vendorAlias, + IReadOnlyCollection purls, + IReadOnlyCollection cpes, + IReadOnlyCollection references) + { + var document = new BsonDocument + { + ["id"] = primaryAlias, + ["vendorId"] = vendorAlias, + ["title"] = $"Simulated advisory {primaryAlias}", + ["summary"] = "Synthetic payload produced by Link-Not-Merge benchmark.", + ["metrics"] = new BsonArray + { + new BsonDocument + { + ["kind"] = "cvss:v3.1", + ["score"] = 7.5, + ["vector"] = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + }, + }, + }; + + if (purls.Count > 0) + { + document["purls"] = new BsonArray(purls); + } + + if (cpes.Count > 0) + { + document["cpes"] = new BsonArray(cpes); + } + + if (references.Count > 0) + { + document["references"] = new BsonArray(references.Select(reference => new BsonDocument + { + ["type"] = reference.Type, + ["url"] = reference.Url, + })); + } + + return document; + } + + private static string ComputeContentHash(BsonDocument rawPayload, string tenant, int group, int revision) + { + using var sha256 = SHA256.Create(); + var seed = $"{tenant}|{group}|{revision}"; + var rawBytes = rawPayload.ToBson(); + var seedBytes = System.Text.Encoding.UTF8.GetBytes(seed); + var combined = new byte[rawBytes.Length + seedBytes.Length]; + Buffer.BlockCopy(rawBytes, 0, combined, 0, rawBytes.Length); + Buffer.BlockCopy(seedBytes, 0, combined, rawBytes.Length, seedBytes.Length); + var hash = sha256.ComputeHash(combined); + return $"sha256:{Convert.ToHexString(hash)}"; + } +} + +internal sealed record ObservationSeed( + string ObservationId, + string Tenant, + string Vendor, + string Stream, + string Api, + string CollectorVersion, + string UpstreamId, + string DocumentVersion, + DateTimeOffset FetchedAt, + DateTimeOffset ReceivedAt, + string ContentHash, + ImmutableArray Aliases, + ImmutableArray Purls, + ImmutableArray Cpes, + ImmutableArray References, + string ContentFormat, + string SpecVersion, + BsonDocument RawPayload) +{ + public BsonDocument ToBsonDocument() + { + var aliases = new BsonArray(Aliases.Select(alias => alias)); + var purls = new BsonArray(Purls.Select(purl => purl)); + var cpes = new BsonArray(Cpes.Select(cpe => cpe)); + var references = new BsonArray(References.Select(reference => new BsonDocument + { + ["type"] = reference.Type, + ["url"] = reference.Url, + })); + + var document = new BsonDocument + { + ["_id"] = ObservationId, + ["tenant"] = Tenant, + ["source"] = new BsonDocument + { + ["vendor"] = Vendor, + ["stream"] = Stream, + ["api"] = Api, + ["collector_version"] = CollectorVersion, + }, + ["upstream"] = new BsonDocument + { + ["upstream_id"] = UpstreamId, + ["document_version"] = DocumentVersion, + ["fetched_at"] = FetchedAt.UtcDateTime, + ["received_at"] = ReceivedAt.UtcDateTime, + ["content_hash"] = ContentHash, + ["signature"] = new BsonDocument + { + ["present"] = false, + ["format"] = BsonNull.Value, + ["key_id"] = BsonNull.Value, + ["signature"] = BsonNull.Value, + }, + }, + ["content"] = new BsonDocument + { + ["format"] = ContentFormat, + ["spec_version"] = SpecVersion, + ["raw"] = RawPayload, + }, + ["identifiers"] = new BsonDocument + { + ["aliases"] = aliases, + ["primary"] = UpstreamId, + ["cve"] = Aliases.FirstOrDefault(alias => alias.StartsWith("CVE-", StringComparison.Ordinal)) ?? UpstreamId, + }, + ["linkset"] = new BsonDocument + { + ["aliases"] = aliases, + ["purls"] = purls, + ["cpes"] = cpes, + ["references"] = references, + ["reconciled_from"] = new BsonArray { "/content/product_tree" }, + }, + ["supersedes"] = BsonNull.Value, + }; + + return document; + } +} + +internal sealed record ObservationReference(string Type, string Url); diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Program.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Program.cs new file mode 100644 index 00000000..68407b03 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Program.cs @@ -0,0 +1,375 @@ +using System.Globalization; +using StellaOps.Bench.LinkNotMerge.Baseline; +using StellaOps.Bench.LinkNotMerge.Reporting; + +namespace StellaOps.Bench.LinkNotMerge; + +internal static class Program +{ + public static async Task Main(string[] args) + { + try + { + var options = ProgramOptions.Parse(args); + var config = await BenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false); + var baseline = await BaselineLoader.LoadAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false); + + var results = new List(); + var reports = new List(); + var failures = new List(); + + foreach (var scenario in config.Scenarios) + { + var iterations = scenario.ResolveIterations(config.Iterations); + var runner = new LinkNotMergeScenarioRunner(scenario); + var execution = runner.Execute(iterations, CancellationToken.None); + + var totalStats = DurationStatistics.From(execution.TotalDurationsMs); + var insertStats = DurationStatistics.From(execution.InsertDurationsMs); + var correlationStats = DurationStatistics.From(execution.CorrelationDurationsMs); + var allocationStats = AllocationStatistics.From(execution.AllocatedMb); + var throughputStats = ThroughputStatistics.From(execution.TotalThroughputsPerSecond); + var mongoThroughputStats = ThroughputStatistics.From(execution.InsertThroughputsPerSecond); + + var thresholdMs = scenario.ThresholdMs ?? options.ThresholdMs ?? config.ThresholdMs; + var throughputFloor = scenario.MinThroughputPerSecond ?? options.MinThroughputPerSecond ?? config.MinThroughputPerSecond; + var mongoThroughputFloor = scenario.MinMongoThroughputPerSecond ?? options.MinMongoThroughputPerSecond ?? config.MinMongoThroughputPerSecond; + var allocationLimit = scenario.MaxAllocatedMb ?? options.MaxAllocatedMb ?? config.MaxAllocatedMb; + + var result = new ScenarioResult( + scenario.ScenarioId, + scenario.DisplayLabel, + iterations, + execution.ObservationCount, + execution.AliasGroups, + execution.LinksetCount, + totalStats, + insertStats, + correlationStats, + throughputStats, + mongoThroughputStats, + allocationStats, + thresholdMs, + throughputFloor, + mongoThroughputFloor, + allocationLimit); + + results.Add(result); + + if (thresholdMs is { } threshold && result.TotalStatistics.MaxMs > threshold) + { + failures.Add($"{result.Id} exceeded total latency threshold: {result.TotalStatistics.MaxMs:F2} ms > {threshold:F2} ms"); + } + + if (throughputFloor is { } floor && result.TotalThroughputStatistics.MinPerSecond < floor) + { + failures.Add($"{result.Id} fell below throughput floor: {result.TotalThroughputStatistics.MinPerSecond:N0} obs/s < {floor:N0} obs/s"); + } + + if (mongoThroughputFloor is { } mongoFloor && result.InsertThroughputStatistics.MinPerSecond < mongoFloor) + { + failures.Add($"{result.Id} fell below Mongo throughput floor: {result.InsertThroughputStatistics.MinPerSecond:N0} ops/s < {mongoFloor:N0} ops/s"); + } + + if (allocationLimit is { } limit && result.AllocationStatistics.MaxAllocatedMb > limit) + { + failures.Add($"{result.Id} exceeded allocation budget: {result.AllocationStatistics.MaxAllocatedMb:F2} MB > {limit:F2} MB"); + } + + baseline.TryGetValue(result.Id, out var baselineEntry); + var report = new BenchmarkScenarioReport(result, baselineEntry, options.RegressionLimit); + reports.Add(report); + failures.AddRange(report.BuildRegressionFailureMessages()); + } + + TablePrinter.Print(results); + + if (!string.IsNullOrWhiteSpace(options.CsvOutPath)) + { + CsvWriter.Write(options.CsvOutPath!, results); + } + + if (!string.IsNullOrWhiteSpace(options.JsonOutPath)) + { + var metadata = new BenchmarkJsonMetadata( + SchemaVersion: "linknotmerge-bench/1.0", + CapturedAtUtc: (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime(), + Commit: options.Commit, + Environment: options.Environment); + + await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath)) + { + PrometheusWriter.Write(options.PrometheusOutPath!, reports); + } + + if (failures.Count > 0) + { + Console.Error.WriteLine(); + Console.Error.WriteLine("Benchmark failures detected:"); + foreach (var failure in failures.Distinct()) + { + Console.Error.WriteLine($" - {failure}"); + } + + return 1; + } + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"linknotmerge-bench error: {ex.Message}"); + return 1; + } + } + + private sealed record ProgramOptions( + string ConfigPath, + int? Iterations, + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MinMongoThroughputPerSecond, + double? MaxAllocatedMb, + string? CsvOutPath, + string? JsonOutPath, + string? PrometheusOutPath, + string BaselinePath, + DateTimeOffset? CapturedAtUtc, + string? Commit, + string? Environment, + double? RegressionLimit) + { + public static ProgramOptions Parse(string[] args) + { + var configPath = DefaultConfigPath(); + var baselinePath = DefaultBaselinePath(); + + int? iterations = null; + double? thresholdMs = null; + double? minThroughput = null; + double? minMongoThroughput = null; + double? maxAllocated = null; + string? csvOut = null; + string? jsonOut = null; + string? promOut = null; + DateTimeOffset? capturedAt = null; + string? commit = null; + string? environment = null; + double? regressionLimit = null; + + for (var index = 0; index < args.Length; index++) + { + var current = args[index]; + switch (current) + { + case "--config": + EnsureNext(args, index); + configPath = Path.GetFullPath(args[++index]); + break; + case "--iterations": + EnsureNext(args, index); + iterations = int.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--threshold-ms": + EnsureNext(args, index); + thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--min-throughput": + EnsureNext(args, index); + minThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--min-mongo-throughput": + EnsureNext(args, index); + minMongoThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--max-allocated-mb": + EnsureNext(args, index); + maxAllocated = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--csv": + EnsureNext(args, index); + csvOut = args[++index]; + break; + case "--json": + EnsureNext(args, index); + jsonOut = args[++index]; + break; + case "--prometheus": + EnsureNext(args, index); + promOut = args[++index]; + break; + case "--baseline": + EnsureNext(args, index); + baselinePath = Path.GetFullPath(args[++index]); + break; + case "--captured-at": + EnsureNext(args, index); + capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); + break; + case "--commit": + EnsureNext(args, index); + commit = args[++index]; + break; + case "--environment": + EnsureNext(args, index); + environment = args[++index]; + break; + case "--regression-limit": + EnsureNext(args, index); + regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--help": + case "-h": + PrintUsage(); + System.Environment.Exit(0); + break; + default: + throw new ArgumentException($"Unknown argument '{current}'."); + } + } + + return new ProgramOptions( + configPath, + iterations, + thresholdMs, + minThroughput, + minMongoThroughput, + maxAllocated, + csvOut, + jsonOut, + promOut, + baselinePath, + capturedAt, + commit, + environment, + regressionLimit); + } + + private static string DefaultConfigPath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "config.json"); + } + + private static string DefaultBaselinePath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "baseline.csv"); + } + + private static void EnsureNext(string[] args, int index) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException("Missing value for argument."); + } + } + + private static void PrintUsage() + { + Console.WriteLine("Usage: linknotmerge-bench [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --config Path to benchmark configuration JSON."); + Console.WriteLine(" --iterations Override iteration count."); + Console.WriteLine(" --threshold-ms Global latency threshold in milliseconds."); + Console.WriteLine(" --min-throughput Global throughput floor (observations/second)."); + Console.WriteLine(" --min-mongo-throughput Mongo insert throughput floor (ops/second)."); + Console.WriteLine(" --max-allocated-mb Global allocation ceiling (MB)."); + Console.WriteLine(" --csv Write CSV results to path."); + Console.WriteLine(" --json Write JSON results to path."); + Console.WriteLine(" --prometheus Write Prometheus exposition metrics to path."); + Console.WriteLine(" --baseline Baseline CSV path."); + Console.WriteLine(" --captured-at Timestamp to embed in JSON metadata."); + Console.WriteLine(" --commit Commit identifier for metadata."); + Console.WriteLine(" --environment Environment label for metadata."); + Console.WriteLine(" --regression-limit Regression multiplier (default 1.15)."); + } + } +} + +internal static class TablePrinter +{ + public static void Print(IEnumerable results) + { + Console.WriteLine("Scenario | Observations | Aliases | Linksets | Total(ms) | Correl(ms) | Insert(ms) | Min k/s | Mongo k/s | Alloc(MB)"); + Console.WriteLine("---------------------------- | ------------- | ------- | -------- | ---------- | ---------- | ----------- | -------- | --------- | --------"); + foreach (var row in results) + { + Console.WriteLine(string.Join(" | ", new[] + { + row.IdColumn, + row.ObservationsColumn, + row.AliasColumn, + row.LinksetColumn, + row.TotalMeanColumn, + row.CorrelationMeanColumn, + row.InsertMeanColumn, + row.ThroughputColumn, + row.MongoThroughputColumn, + row.AllocatedColumn, + })); + } + } +} + +internal static class CsvWriter +{ + public static void Write(string path, IEnumerable results) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(results); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(stream); + writer.WriteLine("scenario,iterations,observations,aliases,linksets,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_throughput_per_sec,min_throughput_per_sec,mean_mongo_throughput_per_sec,min_mongo_throughput_per_sec,max_allocated_mb"); + + foreach (var result in results) + { + writer.Write(result.Id); + writer.Write(','); + writer.Write(result.Iterations.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.ObservationCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.AliasGroups.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.LinksetCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalStatistics.P95Ms.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalStatistics.MaxMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.InsertStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.CorrelationStatistics.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalThroughputStatistics.MeanPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.TotalThroughputStatistics.MinPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.InsertThroughputStatistics.MeanPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.InsertThroughputStatistics.MinPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(result.AllocationStatistics.MaxAllocatedMb.ToString("F4", CultureInfo.InvariantCulture)); + writer.WriteLine(); + } + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Properties/AssemblyInfo.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..66f83687 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Bench.LinkNotMerge.Tests")] diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/BenchmarkJsonWriter.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/BenchmarkJsonWriter.cs new file mode 100644 index 00000000..f1cf6ea7 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/BenchmarkJsonWriter.cs @@ -0,0 +1,151 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.LinkNotMerge.Reporting; + +internal static class BenchmarkJsonWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public static async Task WriteAsync( + string path, + BenchmarkJsonMetadata metadata, + IReadOnlyList reports, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(metadata); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var document = new BenchmarkJsonDocument( + metadata.SchemaVersion, + metadata.CapturedAtUtc, + metadata.Commit, + metadata.Environment, + reports.Select(CreateScenario).ToArray()); + + await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report) + { + var baseline = report.Baseline; + return new BenchmarkJsonScenario( + report.Result.Id, + report.Result.Label, + report.Result.Iterations, + report.Result.ObservationCount, + report.Result.AliasGroups, + report.Result.LinksetCount, + report.Result.TotalStatistics.MeanMs, + report.Result.TotalStatistics.P95Ms, + report.Result.TotalStatistics.MaxMs, + report.Result.InsertStatistics.MeanMs, + report.Result.CorrelationStatistics.MeanMs, + report.Result.TotalThroughputStatistics.MeanPerSecond, + report.Result.TotalThroughputStatistics.MinPerSecond, + report.Result.InsertThroughputStatistics.MeanPerSecond, + report.Result.InsertThroughputStatistics.MinPerSecond, + report.Result.AllocationStatistics.MaxAllocatedMb, + report.Result.ThresholdMs, + report.Result.MinThroughputThresholdPerSecond, + report.Result.MinMongoThroughputThresholdPerSecond, + report.Result.MaxAllocatedThresholdMb, + baseline is null + ? null + : new BenchmarkJsonScenarioBaseline( + baseline.Iterations, + baseline.Observations, + baseline.Aliases, + baseline.Linksets, + baseline.MeanTotalMs, + baseline.P95TotalMs, + baseline.MaxTotalMs, + baseline.MeanInsertMs, + baseline.MeanCorrelationMs, + baseline.MeanThroughputPerSecond, + baseline.MinThroughputPerSecond, + baseline.MeanMongoThroughputPerSecond, + baseline.MinMongoThroughputPerSecond, + baseline.MaxAllocatedMb), + new BenchmarkJsonScenarioRegression( + report.DurationRegressionRatio, + report.ThroughputRegressionRatio, + report.MongoThroughputRegressionRatio, + report.RegressionLimit, + report.RegressionBreached)); + } + + private sealed record BenchmarkJsonDocument( + string SchemaVersion, + DateTimeOffset CapturedAt, + string? Commit, + string? Environment, + IReadOnlyList Scenarios); + + private sealed record BenchmarkJsonScenario( + string Id, + string Label, + int Iterations, + int Observations, + int Aliases, + int Linksets, + double MeanTotalMs, + double P95TotalMs, + double MaxTotalMs, + double MeanInsertMs, + double MeanCorrelationMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MeanMongoThroughputPerSecond, + double MinMongoThroughputPerSecond, + double MaxAllocatedMb, + double? ThresholdMs, + double? MinThroughputThresholdPerSecond, + double? MinMongoThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb, + BenchmarkJsonScenarioBaseline? Baseline, + BenchmarkJsonScenarioRegression Regression); + + private sealed record BenchmarkJsonScenarioBaseline( + int Iterations, + int Observations, + int Aliases, + int Linksets, + double MeanTotalMs, + double P95TotalMs, + double MaxTotalMs, + double MeanInsertMs, + double MeanCorrelationMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MeanMongoThroughputPerSecond, + double MinMongoThroughputPerSecond, + double MaxAllocatedMb); + + private sealed record BenchmarkJsonScenarioRegression( + double? DurationRatio, + double? ThroughputRatio, + double? MongoThroughputRatio, + double Limit, + bool Breached); +} + +internal sealed record BenchmarkJsonMetadata( + string SchemaVersion, + DateTimeOffset CapturedAtUtc, + string? Commit, + string? Environment); diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/BenchmarkScenarioReport.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/BenchmarkScenarioReport.cs new file mode 100644 index 00000000..9da927a2 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/BenchmarkScenarioReport.cs @@ -0,0 +1,89 @@ +using StellaOps.Bench.LinkNotMerge.Baseline; + +namespace StellaOps.Bench.LinkNotMerge.Reporting; + +internal sealed class BenchmarkScenarioReport +{ + private const double DefaultRegressionLimit = 1.15d; + + public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null) + { + Result = result ?? throw new ArgumentNullException(nameof(result)); + Baseline = baseline; + RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : DefaultRegressionLimit; + DurationRegressionRatio = CalculateRatio(result.TotalStatistics.MaxMs, baseline?.MaxTotalMs); + ThroughputRegressionRatio = CalculateInverseRatio(result.TotalThroughputStatistics.MinPerSecond, baseline?.MinThroughputPerSecond); + MongoThroughputRegressionRatio = CalculateInverseRatio(result.InsertThroughputStatistics.MinPerSecond, baseline?.MinMongoThroughputPerSecond); + } + + public ScenarioResult Result { get; } + + public BaselineEntry? Baseline { get; } + + public double RegressionLimit { get; } + + public double? DurationRegressionRatio { get; } + + public double? ThroughputRegressionRatio { get; } + + public double? MongoThroughputRegressionRatio { get; } + + public bool DurationRegressionBreached => DurationRegressionRatio is { } ratio && ratio >= RegressionLimit; + + public bool ThroughputRegressionBreached => ThroughputRegressionRatio is { } ratio && ratio >= RegressionLimit; + + public bool MongoThroughputRegressionBreached => MongoThroughputRegressionRatio is { } ratio && ratio >= RegressionLimit; + + public bool RegressionBreached => DurationRegressionBreached || ThroughputRegressionBreached || MongoThroughputRegressionBreached; + + public IEnumerable BuildRegressionFailureMessages() + { + if (Baseline is null) + { + yield break; + } + + if (DurationRegressionBreached && DurationRegressionRatio is { } durationRatio) + { + var delta = (durationRatio - 1d) * 100d; + yield return $"{Result.Id} exceeded max duration budget: {Result.TotalStatistics.MaxMs:F2} ms vs baseline {Baseline.MaxTotalMs:F2} ms (+{delta:F1}%)."; + } + + if (ThroughputRegressionBreached && ThroughputRegressionRatio is { } throughputRatio) + { + var delta = (throughputRatio - 1d) * 100d; + yield return $"{Result.Id} throughput regressed: min {Result.TotalThroughputStatistics.MinPerSecond:N0} obs/s vs baseline {Baseline.MinThroughputPerSecond:N0} obs/s (-{delta:F1}%)."; + } + + if (MongoThroughputRegressionBreached && MongoThroughputRegressionRatio is { } mongoRatio) + { + var delta = (mongoRatio - 1d) * 100d; + yield return $"{Result.Id} Mongo throughput regressed: min {Result.InsertThroughputStatistics.MinPerSecond:N0} ops/s vs baseline {Baseline.MinMongoThroughputPerSecond:N0} ops/s (-{delta:F1}%)."; + } + } + + private static double? CalculateRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + return current / baseline.Value; + } + + private static double? CalculateInverseRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + if (current <= 0d) + { + return double.PositiveInfinity; + } + + return baseline.Value / current; + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/PrometheusWriter.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/PrometheusWriter.cs new file mode 100644 index 00000000..93a1c571 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/Reporting/PrometheusWriter.cs @@ -0,0 +1,101 @@ +using System.Globalization; +using System.Text; + +namespace StellaOps.Bench.LinkNotMerge.Reporting; + +internal static class PrometheusWriter +{ + public static void Write(string path, IReadOnlyList reports) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var builder = new StringBuilder(); + builder.AppendLine("# HELP linknotmerge_bench_total_ms Link-Not-Merge benchmark total duration metrics (milliseconds)."); + builder.AppendLine("# TYPE linknotmerge_bench_total_ms gauge"); + builder.AppendLine("# HELP linknotmerge_bench_correlation_ms Link-Not-Merge benchmark correlation duration metrics (milliseconds)."); + builder.AppendLine("# TYPE linknotmerge_bench_correlation_ms gauge"); + builder.AppendLine("# HELP linknotmerge_bench_insert_ms Link-Not-Merge benchmark Mongo insert duration metrics (milliseconds)."); + builder.AppendLine("# TYPE linknotmerge_bench_insert_ms gauge"); + builder.AppendLine("# HELP linknotmerge_bench_throughput_per_sec Link-Not-Merge benchmark throughput metrics (observations per second)."); + builder.AppendLine("# TYPE linknotmerge_bench_throughput_per_sec gauge"); + builder.AppendLine("# HELP linknotmerge_bench_mongo_throughput_per_sec Link-Not-Merge benchmark Mongo throughput metrics (operations per second)."); + builder.AppendLine("# TYPE linknotmerge_bench_mongo_throughput_per_sec gauge"); + builder.AppendLine("# HELP linknotmerge_bench_allocated_mb Link-Not-Merge benchmark allocation metrics (megabytes)."); + builder.AppendLine("# TYPE linknotmerge_bench_allocated_mb gauge"); + + foreach (var report in reports) + { + var scenario = Escape(report.Result.Id); + AppendMetric(builder, "linknotmerge_bench_mean_total_ms", scenario, report.Result.TotalStatistics.MeanMs); + AppendMetric(builder, "linknotmerge_bench_p95_total_ms", scenario, report.Result.TotalStatistics.P95Ms); + AppendMetric(builder, "linknotmerge_bench_max_total_ms", scenario, report.Result.TotalStatistics.MaxMs); + AppendMetric(builder, "linknotmerge_bench_threshold_ms", scenario, report.Result.ThresholdMs); + + AppendMetric(builder, "linknotmerge_bench_mean_correlation_ms", scenario, report.Result.CorrelationStatistics.MeanMs); + AppendMetric(builder, "linknotmerge_bench_mean_insert_ms", scenario, report.Result.InsertStatistics.MeanMs); + + AppendMetric(builder, "linknotmerge_bench_mean_throughput_per_sec", scenario, report.Result.TotalThroughputStatistics.MeanPerSecond); + AppendMetric(builder, "linknotmerge_bench_min_throughput_per_sec", scenario, report.Result.TotalThroughputStatistics.MinPerSecond); + AppendMetric(builder, "linknotmerge_bench_throughput_floor_per_sec", scenario, report.Result.MinThroughputThresholdPerSecond); + + AppendMetric(builder, "linknotmerge_bench_mean_mongo_throughput_per_sec", scenario, report.Result.InsertThroughputStatistics.MeanPerSecond); + AppendMetric(builder, "linknotmerge_bench_min_mongo_throughput_per_sec", scenario, report.Result.InsertThroughputStatistics.MinPerSecond); + AppendMetric(builder, "linknotmerge_bench_mongo_throughput_floor_per_sec", scenario, report.Result.MinMongoThroughputThresholdPerSecond); + + AppendMetric(builder, "linknotmerge_bench_max_allocated_mb", scenario, report.Result.AllocationStatistics.MaxAllocatedMb); + AppendMetric(builder, "linknotmerge_bench_max_allocated_threshold_mb", scenario, report.Result.MaxAllocatedThresholdMb); + + if (report.Baseline is { } baseline) + { + AppendMetric(builder, "linknotmerge_bench_baseline_max_total_ms", scenario, baseline.MaxTotalMs); + AppendMetric(builder, "linknotmerge_bench_baseline_min_throughput_per_sec", scenario, baseline.MinThroughputPerSecond); + AppendMetric(builder, "linknotmerge_bench_baseline_min_mongo_throughput_per_sec", scenario, baseline.MinMongoThroughputPerSecond); + } + + if (report.DurationRegressionRatio is { } durationRatio) + { + AppendMetric(builder, "linknotmerge_bench_duration_regression_ratio", scenario, durationRatio); + } + + if (report.ThroughputRegressionRatio is { } throughputRatio) + { + AppendMetric(builder, "linknotmerge_bench_throughput_regression_ratio", scenario, throughputRatio); + } + + if (report.MongoThroughputRegressionRatio is { } mongoRatio) + { + AppendMetric(builder, "linknotmerge_bench_mongo_throughput_regression_ratio", scenario, mongoRatio); + } + + AppendMetric(builder, "linknotmerge_bench_regression_limit", scenario, report.RegressionLimit); + AppendMetric(builder, "linknotmerge_bench_regression_breached", scenario, report.RegressionBreached ? 1 : 0); + } + + File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8); + } + + private static void AppendMetric(StringBuilder builder, string metric, string scenario, double? value) + { + if (!value.HasValue) + { + return; + } + + builder.Append(metric); + builder.Append("{scenario=\""); + builder.Append(scenario); + builder.Append("\"} "); + builder.AppendLine(value.Value.ToString("G17", CultureInfo.InvariantCulture)); + } + + private static string Escape(string value) => + value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioExecutionResult.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioExecutionResult.cs new file mode 100644 index 00000000..e5c1e9af --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioExecutionResult.cs @@ -0,0 +1,14 @@ +namespace StellaOps.Bench.LinkNotMerge; + +internal sealed record ScenarioExecutionResult( + IReadOnlyList TotalDurationsMs, + IReadOnlyList InsertDurationsMs, + IReadOnlyList CorrelationDurationsMs, + IReadOnlyList AllocatedMb, + IReadOnlyList TotalThroughputsPerSecond, + IReadOnlyList InsertThroughputsPerSecond, + int ObservationCount, + int AliasGroups, + int LinksetCount, + int TenantCount, + LinksetAggregationResult AggregationResult); diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioResult.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioResult.cs new file mode 100644 index 00000000..65ec9ffc --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioResult.cs @@ -0,0 +1,42 @@ +using System.Globalization; + +namespace StellaOps.Bench.LinkNotMerge; + +internal sealed record ScenarioResult( + string Id, + string Label, + int Iterations, + int ObservationCount, + int AliasGroups, + int LinksetCount, + DurationStatistics TotalStatistics, + DurationStatistics InsertStatistics, + DurationStatistics CorrelationStatistics, + ThroughputStatistics TotalThroughputStatistics, + ThroughputStatistics InsertThroughputStatistics, + AllocationStatistics AllocationStatistics, + double? ThresholdMs, + double? MinThroughputThresholdPerSecond, + double? MinMongoThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb) +{ + public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28]; + + public string ObservationsColumn => ObservationCount.ToString("N0", CultureInfo.InvariantCulture).PadLeft(12); + + public string AliasColumn => AliasGroups.ToString("N0", CultureInfo.InvariantCulture).PadLeft(8); + + public string LinksetColumn => LinksetCount.ToString("N0", CultureInfo.InvariantCulture).PadLeft(9); + + public string TotalMeanColumn => TotalStatistics.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string CorrelationMeanColumn => CorrelationStatistics.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string InsertMeanColumn => InsertStatistics.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string ThroughputColumn => (TotalThroughputStatistics.MinPerSecond / 1_000d).ToString("F2", CultureInfo.InvariantCulture).PadLeft(11); + + public string MongoThroughputColumn => (InsertThroughputStatistics.MinPerSecond / 1_000d).ToString("F2", CultureInfo.InvariantCulture).PadLeft(11); + + public string AllocatedColumn => AllocationStatistics.MaxAllocatedMb.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9); +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioStatistics.cs b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioStatistics.cs new file mode 100644 index 00000000..f9cd565f --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/ScenarioStatistics.cs @@ -0,0 +1,84 @@ +namespace StellaOps.Bench.LinkNotMerge; + +internal readonly record struct DurationStatistics(double MeanMs, double P95Ms, double MaxMs) +{ + public static DurationStatistics From(IReadOnlyList values) + { + if (values.Count == 0) + { + return new DurationStatistics(0, 0, 0); + } + + var sorted = values.ToArray(); + Array.Sort(sorted); + + var total = 0d; + foreach (var value in values) + { + total += value; + } + + var mean = total / values.Count; + var p95 = Percentile(sorted, 95); + var max = sorted[^1]; + + return new DurationStatistics(mean, p95, max); + } + + private static double Percentile(IReadOnlyList sorted, double percentile) + { + if (sorted.Count == 0) + { + return 0; + } + + var rank = (percentile / 100d) * (sorted.Count - 1); + var lower = (int)Math.Floor(rank); + var upper = (int)Math.Ceiling(rank); + var weight = rank - lower; + + if (upper >= sorted.Count) + { + return sorted[lower]; + } + + return sorted[lower] + weight * (sorted[upper] - sorted[lower]); + } +} + +internal readonly record struct ThroughputStatistics(double MeanPerSecond, double MinPerSecond) +{ + public static ThroughputStatistics From(IReadOnlyList values) + { + if (values.Count == 0) + { + return new ThroughputStatistics(0, 0); + } + + var total = 0d; + var min = double.MaxValue; + + foreach (var value in values) + { + total += value; + min = Math.Min(min, value); + } + + var mean = total / values.Count; + return new ThroughputStatistics(mean, min); + } +} + +internal readonly record struct AllocationStatistics(double MaxAllocatedMb) +{ + public static AllocationStatistics From(IReadOnlyList values) + { + var max = 0d; + foreach (var value in values) + { + max = Math.Max(max, value); + } + + return new AllocationStatistics(max); + } +} diff --git a/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/StellaOps.Bench.LinkNotMerge.csproj b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/StellaOps.Bench.LinkNotMerge.csproj new file mode 100644 index 00000000..7167d07e --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/StellaOps.Bench.LinkNotMerge.csproj @@ -0,0 +1,16 @@ + + + Exe + net10.0 + enable + enable + preview + true + + + + + + + + diff --git a/src/StellaOps.Bench/LinkNotMerge/baseline.csv b/src/StellaOps.Bench/LinkNotMerge/baseline.csv new file mode 100644 index 00000000..7e141e9a --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/baseline.csv @@ -0,0 +1,4 @@ +scenario,iterations,observations,aliases,linksets,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_throughput_per_sec,min_throughput_per_sec,mean_mongo_throughput_per_sec,min_mongo_throughput_per_sec,max_allocated_mb +lnm_ingest_baseline,5,5000,500,6000,555.1984,823.4957,866.6236,366.2635,188.9349,9877.7916,5769.5175,15338.0851,8405.1257,62.4477 +lnm_ingest_fanout_medium,5,10000,800,14800,785.8909,841.6247,842.8815,453.5087,332.3822,12794.9550,11864.0639,22086.0320,20891.0579,145.8328 +lnm_ingest_fanout_high,5,15000,1200,17400,1299.3458,1367.0934,1369.9430,741.6265,557.7193,11571.0991,10949.3607,20232.5180,19781.6762,238.3450 diff --git a/src/StellaOps.Bench/LinkNotMerge/config.json b/src/StellaOps.Bench/LinkNotMerge/config.json new file mode 100644 index 00000000..e3bdbf76 --- /dev/null +++ b/src/StellaOps.Bench/LinkNotMerge/config.json @@ -0,0 +1,57 @@ +{ + "thresholdMs": 2000, + "minThroughputPerSecond": 7000, + "minMongoThroughputPerSecond": 12000, + "maxAllocatedMb": 600, + "iterations": 5, + "scenarios": [ + { + "id": "lnm_ingest_baseline", + "label": "5k observations, 500 aliases", + "observations": 5000, + "aliasGroups": 500, + "purlsPerObservation": 4, + "cpesPerObservation": 2, + "referencesPerObservation": 3, + "tenants": 4, + "batchSize": 250, + "seed": 42022, + "thresholdMs": 900, + "minThroughputPerSecond": 5500, + "minMongoThroughputPerSecond": 8000, + "maxAllocatedMb": 160 + }, + { + "id": "lnm_ingest_fanout_medium", + "label": "10k observations, 800 aliases", + "observations": 10000, + "aliasGroups": 800, + "purlsPerObservation": 6, + "cpesPerObservation": 3, + "referencesPerObservation": 4, + "tenants": 6, + "batchSize": 400, + "seed": 52022, + "thresholdMs": 1300, + "minThroughputPerSecond": 8000, + "minMongoThroughputPerSecond": 13000, + "maxAllocatedMb": 220 + }, + { + "id": "lnm_ingest_fanout_high", + "label": "15k observations, 1200 aliases", + "observations": 15000, + "aliasGroups": 1200, + "purlsPerObservation": 8, + "cpesPerObservation": 4, + "referencesPerObservation": 5, + "tenants": 8, + "batchSize": 500, + "seed": 62022, + "thresholdMs": 2200, + "minThroughputPerSecond": 7000, + "minMongoThroughputPerSecond": 13000, + "maxAllocatedMb": 300 + } + ] +} diff --git a/src/StellaOps.Bench/Notify/README.md b/src/StellaOps.Bench/Notify/README.md new file mode 100644 index 00000000..22d83f52 --- /dev/null +++ b/src/StellaOps.Bench/Notify/README.md @@ -0,0 +1,25 @@ +# Notify Dispatch Bench + +Synthetic workload measuring rule evaluation and channel dispatch fan-out under varying rule densities. + +## Scenarios + +`config.json` defines three density profiles (5%, 20%, 40%). Each scenario synthesizes deterministic tenants, rules, and delivery actions to measure: + +- Latency (mean/p95/max milliseconds) +- Throughput (deliveries per second) +- Managed heap allocations (megabytes) +- Match fan-out statistics (matches and deliveries per event) + +## Running locally + +```bash +dotnet run \ + --project src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/StellaOps.Bench.Notify.csproj \ + -- \ + --csv out/notify-bench.csv \ + --json out/notify-bench.json \ + --prometheus out/notify-bench.prom +``` + +The benchmark exits non-zero if latency exceeds the configured thresholds, throughput drops below the floor, allocations exceed the ceiling, or regression limits are breached relative to `baseline.csv`. diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/BaselineLoaderTests.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/BaselineLoaderTests.cs new file mode 100644 index 00000000..4bf83801 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/BaselineLoaderTests.cs @@ -0,0 +1,38 @@ +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Bench.Notify.Baseline; +using Xunit; + +namespace StellaOps.Bench.Notify.Tests; + +public sealed class BaselineLoaderTests +{ + [Fact] + public async Task LoadAsync_ReadsBaselineEntries() + { + var path = Path.GetTempFileName(); + try + { + await File.WriteAllTextAsync( + path, + "scenario,iterations,events,deliveries,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb\n" + + "notify_dispatch_density_05,5,5000,25000,120.5,150.1,199.9,42000.5,39000.2,85.7\n"); + + var entries = await BaselineLoader.LoadAsync(path, CancellationToken.None); + var entry = Assert.Single(entries); + + Assert.Equal("notify_dispatch_density_05", entry.Key); + Assert.Equal(5, entry.Value.Iterations); + Assert.Equal(5000, entry.Value.EventCount); + Assert.Equal(25000, entry.Value.DeliveryCount); + Assert.Equal(120.5, entry.Value.MeanMs); + Assert.Equal(39000.2, entry.Value.MinThroughputPerSecond); + Assert.Equal(85.7, entry.Value.MaxAllocatedMb); + } + finally + { + File.Delete(path); + } + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/BenchmarkScenarioReportTests.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/BenchmarkScenarioReportTests.cs new file mode 100644 index 00000000..5ac43b76 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/BenchmarkScenarioReportTests.cs @@ -0,0 +1,85 @@ +using System.Linq; +using StellaOps.Bench.Notify.Baseline; +using StellaOps.Bench.Notify.Reporting; +using Xunit; + +namespace StellaOps.Bench.Notify.Tests; + +public sealed class BenchmarkScenarioReportTests +{ + [Fact] + public void RegressionDetection_FlagsLatencies() + { + var result = new ScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + TotalEvents: 1000, + TotalRules: 100, + ActionsPerRule: 2, + AverageMatchesPerEvent: 10, + MinMatchesPerEvent: 8, + MaxMatchesPerEvent: 12, + AverageDeliveriesPerEvent: 20, + TotalDeliveries: 20000, + MeanMs: 200, + P95Ms: 250, + MaxMs: 300, + MeanThroughputPerSecond: 50000, + MinThroughputPerSecond: 40000, + MaxAllocatedMb: 100, + ThresholdMs: null, + MinThroughputThresholdPerSecond: null, + MaxAllocatedThresholdMb: null); + + var baseline = new BaselineEntry( + ScenarioId: "scenario", + Iterations: 3, + EventCount: 1000, + DeliveryCount: 20000, + MeanMs: 150, + P95Ms: 180, + MaxMs: 200, + MeanThroughputPerSecond: 60000, + MinThroughputPerSecond: 50000, + MaxAllocatedMb: 90); + + var report = new BenchmarkScenarioReport(result, baseline, regressionLimit: 1.1); + + Assert.True(report.DurationRegressionBreached); + Assert.True(report.ThroughputRegressionBreached); + Assert.Contains(report.BuildRegressionFailureMessages(), message => message.Contains("max duration")); + } + + [Fact] + public void RegressionDetection_NoBaseline_NoBreaches() + { + var result = new ScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + TotalEvents: 1000, + TotalRules: 100, + ActionsPerRule: 2, + AverageMatchesPerEvent: 10, + MinMatchesPerEvent: 8, + MaxMatchesPerEvent: 12, + AverageDeliveriesPerEvent: 20, + TotalDeliveries: 20000, + MeanMs: 200, + P95Ms: 250, + MaxMs: 300, + MeanThroughputPerSecond: 50000, + MinThroughputPerSecond: 40000, + MaxAllocatedMb: 100, + ThresholdMs: null, + MinThroughputThresholdPerSecond: null, + MaxAllocatedThresholdMb: null); + + var report = new BenchmarkScenarioReport(result, baseline: null, regressionLimit: null); + + Assert.False(report.DurationRegressionBreached); + Assert.False(report.ThroughputRegressionBreached); + Assert.Empty(report.BuildRegressionFailureMessages()); + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/NotifyScenarioRunnerTests.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/NotifyScenarioRunnerTests.cs new file mode 100644 index 00000000..2df58760 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/NotifyScenarioRunnerTests.cs @@ -0,0 +1,33 @@ +using System.Threading; +using Xunit; + +namespace StellaOps.Bench.Notify.Tests; + +public sealed class NotifyScenarioRunnerTests +{ + [Fact] + public void Execute_ComputesDeterministicMetrics() + { + var config = new NotifyScenarioConfig + { + Id = "unit_test", + EventCount = 500, + RuleCount = 40, + ActionsPerRule = 3, + MatchRate = 0.25, + TenantCount = 4, + ChannelCount = 16 + }; + + var runner = new NotifyScenarioRunner(config); + var result = runner.Execute(2, CancellationToken.None); + + Assert.Equal(config.ResolveEventCount(), result.TotalEvents); + Assert.Equal(config.ResolveRuleCount(), result.TotalRules); + Assert.Equal(config.ResolveActionsPerRule(), result.ActionsPerRule); + Assert.True(result.TotalMatches > 0); + Assert.Equal(result.TotalMatches * result.ActionsPerRule, result.TotalDeliveries); + Assert.Equal(2, result.Durations.Count); + Assert.All(result.Durations, value => Assert.True(value > 0)); + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/PrometheusWriterTests.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/PrometheusWriterTests.cs new file mode 100644 index 00000000..309f85e3 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/PrometheusWriterTests.cs @@ -0,0 +1,64 @@ +using System.IO; +using StellaOps.Bench.Notify.Baseline; +using StellaOps.Bench.Notify.Reporting; +using Xunit; + +namespace StellaOps.Bench.Notify.Tests; + +public sealed class PrometheusWriterTests +{ + [Fact] + public void Write_EmitsScenarioMetrics() + { + var result = new ScenarioResult( + Id: "scenario", + Label: "Scenario", + Iterations: 3, + TotalEvents: 1000, + TotalRules: 100, + ActionsPerRule: 2, + AverageMatchesPerEvent: 10, + MinMatchesPerEvent: 8, + MaxMatchesPerEvent: 12, + AverageDeliveriesPerEvent: 20, + TotalDeliveries: 20000, + MeanMs: 200, + P95Ms: 250, + MaxMs: 300, + MeanThroughputPerSecond: 50000, + MinThroughputPerSecond: 40000, + MaxAllocatedMb: 100, + ThresholdMs: 900, + MinThroughputThresholdPerSecond: 35000, + MaxAllocatedThresholdMb: 150); + + var baseline = new BaselineEntry( + ScenarioId: "scenario", + Iterations: 3, + EventCount: 1000, + DeliveryCount: 20000, + MeanMs: 180, + P95Ms: 210, + MaxMs: 240, + MeanThroughputPerSecond: 52000, + MinThroughputPerSecond: 41000, + MaxAllocatedMb: 95); + + var report = new BenchmarkScenarioReport(result, baseline); + + var path = Path.GetTempFileName(); + try + { + PrometheusWriter.Write(path, new[] { report }); + var content = File.ReadAllText(path); + + Assert.Contains("notify_dispatch_bench_mean_ms", content); + Assert.Contains("scenario\"} 200", content); + Assert.Contains("notify_dispatch_bench_baseline_mean_ms", content); + } + finally + { + File.Delete(path); + } + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/StellaOps.Bench.Notify.Tests.csproj b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/StellaOps.Bench.Notify.Tests.csproj new file mode 100644 index 00000000..f7540b8e --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/StellaOps.Bench.Notify.Tests.csproj @@ -0,0 +1,27 @@ + + + net10.0 + enable + enable + preview + true + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Baseline/BaselineEntry.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Baseline/BaselineEntry.cs new file mode 100644 index 00000000..c03e8b49 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Baseline/BaselineEntry.cs @@ -0,0 +1,13 @@ +namespace StellaOps.Bench.Notify.Baseline; + +internal sealed record BaselineEntry( + string ScenarioId, + int Iterations, + int EventCount, + int DeliveryCount, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb); diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Baseline/BaselineLoader.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Baseline/BaselineLoader.cs new file mode 100644 index 00000000..792880da --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Baseline/BaselineLoader.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Bench.Notify.Baseline; + +internal static class BaselineLoader +{ + public static async Task> LoadAsync(string path, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var results = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read); + using var reader = new StreamReader(stream); + + var lineNumber = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var line = await reader.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + lineNumber++; + if (lineNumber == 1 || string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = line.Split(',', StringSplitOptions.TrimEntries); + if (parts.Length < 10) + { + throw new InvalidOperationException($"Baseline '{resolved}' line {lineNumber} is invalid (expected 10 columns, found {parts.Length})."); + } + + var entry = new BaselineEntry( + ScenarioId: parts[0], + Iterations: ParseInt(parts[1], resolved, lineNumber), + EventCount: ParseInt(parts[2], resolved, lineNumber), + DeliveryCount: ParseInt(parts[3], resolved, lineNumber), + MeanMs: ParseDouble(parts[4], resolved, lineNumber), + P95Ms: ParseDouble(parts[5], resolved, lineNumber), + MaxMs: ParseDouble(parts[6], resolved, lineNumber), + MeanThroughputPerSecond: ParseDouble(parts[7], resolved, lineNumber), + MinThroughputPerSecond: ParseDouble(parts[8], resolved, lineNumber), + MaxAllocatedMb: ParseDouble(parts[9], resolved, lineNumber)); + + results[entry.ScenarioId] = entry; + } + + return results; + } + + private static int ParseInt(string value, string file, int line) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid integer '{value}'."); + } + + private static double ParseDouble(string value, string file, int line) + { + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid number '{value}'."); + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/BenchmarkConfig.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/BenchmarkConfig.cs new file mode 100644 index 00000000..49b53a31 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/BenchmarkConfig.cs @@ -0,0 +1,220 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.Notify; + +internal sealed record BenchmarkConfig( + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MaxAllocatedMb, + int? Iterations, + IReadOnlyList Scenarios) +{ + public static async Task LoadAsync(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + throw new FileNotFoundException($"Benchmark configuration '{resolved}' was not found.", resolved); + } + + await using var stream = File.OpenRead(resolved); + var model = await JsonSerializer.DeserializeAsync( + stream, + new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }).ConfigureAwait(false); + + if (model is null) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' could not be parsed."); + } + + if (model.Scenarios.Count == 0) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' does not contain any scenarios."); + } + + foreach (var scenario in model.Scenarios) + { + scenario.Validate(); + } + + return new BenchmarkConfig( + model.ThresholdMs, + model.MinThroughputPerSecond, + model.MaxAllocatedMb, + model.Iterations, + model.Scenarios); + } + + private sealed class BenchmarkConfigModel + { + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("scenarios")] + public List Scenarios { get; init; } = new(); + } +} + +internal sealed class NotifyScenarioConfig +{ + private const int DefaultEventCount = 10_000; + private const int DefaultRuleCount = 200; + private const int DefaultActionsPerRule = 3; + private const double DefaultMatchRate = 0.25d; + private const int DefaultTenantCount = 4; + private const int DefaultChannelCount = 8; + private const int BaseSeed = 2025_10_26; + + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } + + [JsonPropertyName("eventCount")] + public int EventCount { get; init; } = DefaultEventCount; + + [JsonPropertyName("ruleCount")] + public int RuleCount { get; init; } = DefaultRuleCount; + + [JsonPropertyName("actionsPerRule")] + public int ActionsPerRule { get; init; } = DefaultActionsPerRule; + + [JsonPropertyName("matchRate")] + public double? MatchRate { get; init; } + + [JsonPropertyName("tenantCount")] + public int? TenantCount { get; init; } + + [JsonPropertyName("channelCount")] + public int? ChannelCount { get; init; } + + [JsonPropertyName("seed")] + public int? Seed { get; init; } + + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + public string ScenarioId => string.IsNullOrWhiteSpace(Id) ? "notify_dispatch" : Id!.Trim(); + + public string DisplayLabel => string.IsNullOrWhiteSpace(Label) ? ScenarioId : Label!.Trim(); + + public int ResolveEventCount() + { + if (EventCount <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires eventCount > 0."); + } + + return EventCount; + } + + public int ResolveRuleCount() + { + if (RuleCount <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires ruleCount > 0."); + } + + return RuleCount; + } + + public int ResolveActionsPerRule() + { + if (ActionsPerRule <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires actionsPerRule > 0."); + } + + return ActionsPerRule; + } + + public double ResolveMatchRate() + { + var rate = MatchRate ?? DefaultMatchRate; + if (!double.IsFinite(rate) || rate <= 0d || rate > 1d) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires matchRate within (0, 1]."); + } + + return rate; + } + + public int ResolveTenantCount() + { + var tenants = TenantCount ?? DefaultTenantCount; + if (tenants <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires tenantCount > 0."); + } + + return tenants; + } + + public int ResolveChannelCount() + { + var channels = ChannelCount ?? DefaultChannelCount; + if (channels <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires channelCount > 0."); + } + + return channels; + } + + public int ResolveSeed() + { + if (Seed is { } explicitSeed && explicitSeed > 0) + { + return explicitSeed; + } + + var material = Encoding.UTF8.GetBytes($"stellaops-notify-bench::{ScenarioId}"); + var hash = SHA256.HashData(material); + var derived = BitConverter.ToInt32(hash, 0) & int.MaxValue; + if (derived == 0) + { + derived = BaseSeed; + } + + return derived; + } + + public void Validate() + { + ResolveEventCount(); + ResolveRuleCount(); + ResolveActionsPerRule(); + ResolveMatchRate(); + ResolveTenantCount(); + ResolveChannelCount(); + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/DispatchAccumulator.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/DispatchAccumulator.cs new file mode 100644 index 00000000..9080ff00 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/DispatchAccumulator.cs @@ -0,0 +1,26 @@ +using System; + +namespace StellaOps.Bench.Notify; + +internal sealed class DispatchAccumulator +{ + private long _value = 17; + + public void Add(int ruleHash, int actionHash, int eventHash) + { + unchecked + { + _value = (_value * 31) ^ ruleHash; + _value = (_value * 31) ^ actionHash; + _value = (_value * 31) ^ eventHash; + } + } + + public void AssertConsumed() + { + if (_value == 17) + { + throw new InvalidOperationException("Dispatch accumulator did not receive any values."); + } + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/NotifyScenarioRunner.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/NotifyScenarioRunner.cs new file mode 100644 index 00000000..2b4739b5 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/NotifyScenarioRunner.cs @@ -0,0 +1,386 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using StellaOps.Notify.Models; + +namespace StellaOps.Bench.Notify; + +internal sealed class NotifyScenarioRunner +{ + private static readonly DateTimeOffset BaseTimestamp = new(2025, 10, 26, 0, 0, 0, TimeSpan.Zero); + private const string EventKind = NotifyEventKinds.ScannerReportReady; + + private readonly NotifyScenarioConfig _config; + private readonly EventDescriptor[] _events; + private readonly RuleDescriptor[][] _rulesByTenant; + private readonly int _totalEvents; + private readonly int _ruleCount; + private readonly int _actionsPerRule; + private readonly int _totalMatches; + private readonly int _totalDeliveries; + private readonly double _averageMatchesPerEvent; + private readonly double _averageDeliveriesPerEvent; + private readonly int _minMatchesPerEvent; + private readonly int _maxMatchesPerEvent; + + public NotifyScenarioRunner(NotifyScenarioConfig config) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + + var eventCount = config.ResolveEventCount(); + var ruleCount = config.ResolveRuleCount(); + var actionsPerRule = config.ResolveActionsPerRule(); + var matchRate = config.ResolveMatchRate(); + var tenantCount = config.ResolveTenantCount(); + var channelCount = config.ResolveChannelCount(); + var seed = config.ResolveSeed(); + + if (tenantCount > ruleCount) + { + tenantCount = Math.Max(1, ruleCount); + } + + _totalEvents = eventCount; + _ruleCount = ruleCount; + _actionsPerRule = actionsPerRule; + + var tenants = BuildTenants(tenantCount); + var channels = BuildChannels(channelCount); + var random = new Random(seed); + + var targetMatchesPerEvent = Math.Max(1, (int)Math.Round(ruleCount * matchRate)); + targetMatchesPerEvent = Math.Min(targetMatchesPerEvent, ruleCount); + + var ruleDescriptors = new List(ruleCount); + var groups = new List(); + + var ruleIndex = 0; + var groupIndex = 0; + var channelCursor = 0; + + while (ruleIndex < ruleCount) + { + var groupSize = Math.Min(targetMatchesPerEvent, ruleCount - ruleIndex); + var tenantIndex = groupIndex % tenantCount; + var tenantId = tenants[tenantIndex]; + + var namespaceValue = $"svc-{tenantIndex:D2}-{groupIndex:D3}"; + var repositoryValue = $"registry.local/{tenantId}/service-{groupIndex:D3}"; + var digestValue = GenerateDigest(random, groupIndex); + + var rules = new RuleDescriptor[groupSize]; + for (var local = 0; local < groupSize && ruleIndex < ruleCount; local++, ruleIndex++) + { + var ruleId = $"rule-{groupIndex:D3}-{local:D3}"; + var actions = new ActionDescriptor[actionsPerRule]; + + for (var actionIndex = 0; actionIndex < actionsPerRule; actionIndex++) + { + var channel = channels[channelCursor % channelCount]; + channelCursor++; + + var actionId = $"{ruleId}-act-{actionIndex:D2}"; + actions[actionIndex] = new ActionDescriptor( + actionId, + channel, + StableHash($"{actionId}|{channel}")); + } + + rules[local] = new RuleDescriptor( + ruleId, + StableHash(ruleId), + tenantIndex, + namespaceValue, + repositoryValue, + digestValue, + actions); + + ruleDescriptors.Add(rules[local]); + } + + groups.Add(new RuleGroup(tenantIndex, namespaceValue, repositoryValue, digestValue, rules)); + groupIndex++; + } + + _rulesByTenant = BuildRulesByTenant(tenantCount, ruleDescriptors); + + var events = new EventDescriptor[eventCount]; + long totalMatches = 0; + var minMatches = int.MaxValue; + var maxMatches = 0; + + for (var eventIndex = 0; eventIndex < eventCount; eventIndex++) + { + var group = groups[eventIndex % groups.Count]; + var matchingRules = group.Rules.Length; + + totalMatches += matchingRules; + if (matchingRules < minMatches) + { + minMatches = matchingRules; + } + + if (matchingRules > maxMatches) + { + maxMatches = matchingRules; + } + + var eventId = GenerateEventId(random, group.TenantIndex, eventIndex); + var timestamp = BaseTimestamp.AddMilliseconds(eventIndex * 10d); + + // Materialize NotifyEvent to reflect production payload shape. + _ = NotifyEvent.Create( + eventId, + EventKind, + tenants[group.TenantIndex], + timestamp, + payload: null, + scope: NotifyEventScope.Create( + @namespace: group.Namespace, + repo: group.Repository, + digest: group.Digest)); + + events[eventIndex] = new EventDescriptor( + group.TenantIndex, + EventKind, + group.Namespace, + group.Repository, + group.Digest, + ComputeEventHash(eventId)); + } + + _events = events; + _totalMatches = checked((int)totalMatches); + _totalDeliveries = checked(_totalMatches * actionsPerRule); + _averageMatchesPerEvent = totalMatches / (double)eventCount; + _averageDeliveriesPerEvent = _averageMatchesPerEvent * actionsPerRule; + _minMatchesPerEvent = minMatches; + _maxMatchesPerEvent = maxMatches; + } + + public ScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken) + { + if (iterations <= 0) + { + throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); + } + + var durations = new double[iterations]; + var throughputs = new double[iterations]; + var allocations = new double[iterations]; + + for (var index = 0; index < iterations; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var beforeAllocated = GC.GetTotalAllocatedBytes(); + var stopwatch = Stopwatch.StartNew(); + + var accumulator = new DispatchAccumulator(); + var observedMatches = 0; + var observedDeliveries = 0; + + foreach (ref readonly var @event in _events.AsSpan()) + { + var tenantRules = _rulesByTenant[@event.TenantIndex]; + foreach (var rule in tenantRules) + { + if (!Matches(rule, @event)) + { + continue; + } + + observedMatches++; + + var actions = rule.Actions; + for (var actionIndex = 0; actionIndex < actions.Length; actionIndex++) + { + observedDeliveries++; + accumulator.Add(rule.RuleHash, actions[actionIndex].Hash, @event.EventHash); + } + } + } + + stopwatch.Stop(); + + if (observedMatches != _totalMatches) + { + throw new InvalidOperationException($"Scenario '{_config.ScenarioId}' expected {_totalMatches} matches but observed {observedMatches}."); + } + + if (observedDeliveries != _totalDeliveries) + { + throw new InvalidOperationException($"Scenario '{_config.ScenarioId}' expected {_totalDeliveries} deliveries but observed {observedDeliveries}."); + } + + accumulator.AssertConsumed(); + + var elapsedMs = stopwatch.Elapsed.TotalMilliseconds; + if (elapsedMs <= 0d) + { + elapsedMs = 0.0001d; + } + + var afterAllocated = GC.GetTotalAllocatedBytes(); + + durations[index] = elapsedMs; + throughputs[index] = observedDeliveries / Math.Max(stopwatch.Elapsed.TotalSeconds, 0.0001d); + allocations[index] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d); + } + + return new ScenarioExecutionResult( + durations, + throughputs, + allocations, + _totalEvents, + _ruleCount, + _actionsPerRule, + _averageMatchesPerEvent, + _minMatchesPerEvent, + _maxMatchesPerEvent, + _averageDeliveriesPerEvent, + _totalMatches, + _totalDeliveries); + } + + private static bool Matches(in RuleDescriptor rule, in EventDescriptor @event) + { + if (!string.Equals(@event.Kind, EventKind, StringComparison.Ordinal)) + { + return false; + } + + if (!string.Equals(rule.Namespace, @event.Namespace, StringComparison.Ordinal)) + { + return false; + } + + if (!string.Equals(rule.Repository, @event.Repository, StringComparison.Ordinal)) + { + return false; + } + + if (!string.Equals(rule.Digest, @event.Digest, StringComparison.Ordinal)) + { + return false; + } + + return true; + } + + private static int ComputeEventHash(Guid eventId) + { + var bytes = eventId.ToByteArray(); + var value = BitConverter.ToInt32(bytes, 0); + return value & int.MaxValue; + } + + private static string GenerateDigest(Random random, int groupIndex) + { + var buffer = new byte[16]; + random.NextBytes(buffer); + + var hex = Convert.ToHexString(buffer).ToLowerInvariant(); + return $"sha256:{hex}{groupIndex:D3}"; + } + + private static Guid GenerateEventId(Random random, int tenantIndex, int eventIndex) + { + Span buffer = stackalloc byte[16]; + random.NextBytes(buffer); + buffer[^1] = (byte)(tenantIndex & 0xFF); + buffer[^2] = (byte)(eventIndex & 0xFF); + return new Guid(buffer); + } + + private static RuleDescriptor[][] BuildRulesByTenant(int tenantCount, List rules) + { + var result = new RuleDescriptor[tenantCount][]; + for (var tenantIndex = 0; tenantIndex < tenantCount; tenantIndex++) + { + result[tenantIndex] = rules + .Where(rule => rule.TenantIndex == tenantIndex) + .ToArray(); + } + + return result; + } + + private static string[] BuildTenants(int tenantCount) + { + var tenants = new string[tenantCount]; + for (var index = 0; index < tenantCount; index++) + { + tenants[index] = $"tenant-{index:D2}"; + } + + return tenants; + } + + private static string[] BuildChannels(int channelCount) + { + var channels = new string[channelCount]; + for (var index = 0; index < channelCount; index++) + { + var kind = (index % 4) switch + { + 0 => "slack", + 1 => "teams", + 2 => "email", + _ => "webhook" + }; + + channels[index] = $"{kind}:channel-{index:D2}"; + } + + return channels; + } + + private static int StableHash(string value) + { + unchecked + { + const int offset = unchecked((int)2166136261); + const int prime = 16777619; + + var hash = offset; + foreach (var ch in value.AsSpan()) + { + hash ^= ch; + hash *= prime; + } + + return hash & int.MaxValue; + } + } + + private readonly record struct RuleDescriptor( + string RuleId, + int RuleHash, + int TenantIndex, + string Namespace, + string Repository, + string Digest, + ActionDescriptor[] Actions); + + private readonly record struct ActionDescriptor( + string ActionId, + string Channel, + int Hash); + + private readonly record struct RuleGroup( + int TenantIndex, + string Namespace, + string Repository, + string Digest, + RuleDescriptor[] Rules); + + private readonly record struct EventDescriptor( + int TenantIndex, + string Kind, + string Namespace, + string Repository, + string Digest, + int EventHash); +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Program.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Program.cs new file mode 100644 index 00000000..5264b3f3 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Program.cs @@ -0,0 +1,364 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Globalization; +using StellaOps.Bench.Notify.Baseline; +using StellaOps.Bench.Notify.Reporting; + +namespace StellaOps.Bench.Notify; + +internal static class Program +{ + public static async Task Main(string[] args) + { + try + { + var options = ProgramOptions.Parse(args); + var config = await BenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false); + + var baseline = await BaselineLoader.LoadAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false); + + var results = new List(); + var reports = new List(); + var failures = new List(); + + foreach (var scenario in config.Scenarios) + { + var iterations = options.Iterations + ?? scenario.Iterations + ?? config.Iterations + ?? 5; + + var runner = new NotifyScenarioRunner(scenario); + var execution = runner.Execute(iterations, CancellationToken.None); + + var durationStats = DurationStatistics.From(execution.Durations); + var throughputStats = ThroughputStatistics.From(execution.Throughputs); + var allocationStats = AllocationStatistics.From(execution.AllocatedMb); + + var scenarioThreshold = scenario.ThresholdMs ?? options.ThresholdMs ?? config.ThresholdMs; + var scenarioThroughputFloor = scenario.MinThroughputPerSecond ?? options.MinThroughputPerSecond ?? config.MinThroughputPerSecond; + var scenarioAllocationLimit = scenario.MaxAllocatedMb ?? options.MaxAllocatedMb ?? config.MaxAllocatedMb; + + var result = new ScenarioResult( + scenario.ScenarioId, + scenario.DisplayLabel, + iterations, + execution.TotalEvents, + execution.TotalRules, + execution.ActionsPerRule, + execution.AverageMatchesPerEvent, + execution.MinMatchesPerEvent, + execution.MaxMatchesPerEvent, + execution.AverageDeliveriesPerEvent, + execution.TotalDeliveries, + durationStats.MeanMs, + durationStats.P95Ms, + durationStats.MaxMs, + throughputStats.MeanPerSecond, + throughputStats.MinPerSecond, + allocationStats.MaxAllocatedMb, + scenarioThreshold, + scenarioThroughputFloor, + scenarioAllocationLimit); + + results.Add(result); + + if (scenarioThreshold is { } threshold && result.MaxMs > threshold) + { + failures.Add($"{result.Id} exceeded latency threshold: {result.MaxMs:F2} ms > {threshold:F2} ms"); + } + + if (scenarioThroughputFloor is { } floor && result.MinThroughputPerSecond < floor) + { + failures.Add($"{result.Id} fell below throughput floor: {result.MinThroughputPerSecond:N0} deliveries/s < {floor:N0} deliveries/s"); + } + + if (scenarioAllocationLimit is { } limit && result.MaxAllocatedMb > limit) + { + failures.Add($"{result.Id} exceeded allocation budget: {result.MaxAllocatedMb:F2} MB > {limit:F2} MB"); + } + + baseline.TryGetValue(result.Id, out var baselineEntry); + var report = new BenchmarkScenarioReport(result, baselineEntry, options.RegressionLimit); + reports.Add(report); + failures.AddRange(report.BuildRegressionFailureMessages()); + } + + TablePrinter.Print(results); + + if (!string.IsNullOrWhiteSpace(options.CsvOutPath)) + { + CsvWriter.Write(options.CsvOutPath!, results); + } + + if (!string.IsNullOrWhiteSpace(options.JsonOutPath)) + { + var metadata = new BenchmarkJsonMetadata( + SchemaVersion: "notify-dispatch-bench/1.0", + CapturedAtUtc: (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime(), + Commit: options.Commit, + Environment: options.Environment); + + await BenchmarkJsonWriter.WriteAsync( + options.JsonOutPath!, + metadata, + reports, + CancellationToken.None).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath)) + { + PrometheusWriter.Write(options.PrometheusOutPath!, reports); + } + + if (failures.Count > 0) + { + Console.Error.WriteLine(); + Console.Error.WriteLine("Benchmark failures detected:"); + foreach (var failure in failures.Distinct()) + { + Console.Error.WriteLine($" - {failure}"); + } + + return 1; + } + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"notify-bench error: {ex.Message}"); + return 1; + } + } + + private sealed record ProgramOptions( + string ConfigPath, + int? Iterations, + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MaxAllocatedMb, + string? CsvOutPath, + string? JsonOutPath, + string? PrometheusOutPath, + string BaselinePath, + DateTimeOffset? CapturedAtUtc, + string? Commit, + string? Environment, + double? RegressionLimit) + { + public static ProgramOptions Parse(string[] args) + { + var configPath = DefaultConfigPath(); + var baselinePath = DefaultBaselinePath(); + + int? iterations = null; + double? thresholdMs = null; + double? minThroughput = null; + double? maxAllocated = null; + string? csvOut = null; + string? jsonOut = null; + string? promOut = null; + DateTimeOffset? capturedAt = null; + string? commit = null; + string? environment = null; + double? regressionLimit = null; + + for (var index = 0; index < args.Length; index++) + { + var current = args[index]; + switch (current) + { + case "--config": + EnsureNext(args, index); + configPath = Path.GetFullPath(args[++index]); + break; + case "--iterations": + EnsureNext(args, index); + iterations = int.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--threshold-ms": + EnsureNext(args, index); + thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--min-throughput": + EnsureNext(args, index); + minThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--max-allocated-mb": + EnsureNext(args, index); + maxAllocated = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--csv": + EnsureNext(args, index); + csvOut = args[++index]; + break; + case "--json": + EnsureNext(args, index); + jsonOut = args[++index]; + break; + case "--prometheus": + EnsureNext(args, index); + promOut = args[++index]; + break; + case "--baseline": + EnsureNext(args, index); + baselinePath = Path.GetFullPath(args[++index]); + break; + case "--captured-at": + EnsureNext(args, index); + capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); + break; + case "--commit": + EnsureNext(args, index); + commit = args[++index]; + break; + case "--environment": + EnsureNext(args, index); + environment = args[++index]; + break; + case "--regression-limit": + EnsureNext(args, index); + regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--help": + case "-h": + PrintUsage(); + System.Environment.Exit(0); + break; + default: + throw new ArgumentException($"Unknown argument '{current}'."); + } + } + + return new ProgramOptions( + configPath, + iterations, + thresholdMs, + minThroughput, + maxAllocated, + csvOut, + jsonOut, + promOut, + baselinePath, + capturedAt, + commit, + environment, + regressionLimit); + } + + private static string DefaultConfigPath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "config.json"); + } + + private static string DefaultBaselinePath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "baseline.csv"); + } + + private static void EnsureNext(string[] args, int index) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException("Missing value for argument."); + } + } + + private static void PrintUsage() + { + Console.WriteLine("Usage: notify-bench [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --config Path to benchmark configuration JSON."); + Console.WriteLine(" --iterations Override iteration count."); + Console.WriteLine(" --threshold-ms Global latency threshold in milliseconds."); + Console.WriteLine(" --min-throughput Global throughput floor (deliveries/second)."); + Console.WriteLine(" --max-allocated-mb Global allocation ceiling (MB)."); + Console.WriteLine(" --csv Write CSV results to path."); + Console.WriteLine(" --json Write JSON results to path."); + Console.WriteLine(" --prometheus Write Prometheus exposition metrics to path."); + Console.WriteLine(" --baseline Baseline CSV path."); + Console.WriteLine(" --captured-at Timestamp to embed in JSON metadata."); + Console.WriteLine(" --commit Commit identifier for metadata."); + Console.WriteLine(" --environment Environment label for metadata."); + Console.WriteLine(" --regression-limit Regression multiplier (default 1.15)."); + } + } +} + +internal static class TablePrinter +{ + public static void Print(IEnumerable results) + { + Console.WriteLine("Scenario | Events | Rules | Match/Evt | Deliver/Evt | Mean(ms) | P95(ms) | Max(ms) | Min k/s | Alloc(MB)"); + Console.WriteLine("---------------------------- | ------------| -------- | --------- | ----------- | ---------- | ---------- | ---------- | -------- | --------"); + foreach (var row in results) + { + Console.WriteLine(string.Join(" | ", new[] + { + row.IdColumn, + row.EventsColumn, + row.RulesColumn, + row.MatchesColumn, + row.DeliveriesColumn, + row.MeanColumn, + row.P95Column, + row.MaxColumn, + row.MinThroughputColumn, + row.AllocatedColumn + })); + } + } +} + +internal static class CsvWriter +{ + public static void Write(string path, IEnumerable results) + { + var resolvedPath = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolvedPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + using var stream = new FileStream(resolvedPath, FileMode.Create, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(stream); + writer.WriteLine("scenario,iterations,events,deliveries,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb"); + + foreach (var row in results) + { + writer.Write(row.Id); + writer.Write(','); + writer.Write(row.Iterations.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.TotalEvents.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.TotalDeliveries.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.P95Ms.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MaxMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MeanThroughputPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MinThroughputPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MaxAllocatedMb.ToString("F4", CultureInfo.InvariantCulture)); + writer.WriteLine(); + } + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Properties/AssemblyInfo.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..99c056c4 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Bench.Notify.Tests")] diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/BenchmarkJsonWriter.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/BenchmarkJsonWriter.cs new file mode 100644 index 00000000..b8ea781a --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/BenchmarkJsonWriter.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Bench.Notify.Baseline; + +namespace StellaOps.Bench.Notify.Reporting; + +internal static class BenchmarkJsonWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + public static async Task WriteAsync( + string path, + BenchmarkJsonMetadata metadata, + IReadOnlyList reports, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(metadata); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var document = new BenchmarkJsonDocument( + metadata.SchemaVersion, + metadata.CapturedAtUtc, + metadata.Commit, + metadata.Environment, + reports.Select(CreateScenario).ToArray()); + + await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report) + { + var baseline = report.Baseline; + + return new BenchmarkJsonScenario( + report.Result.Id, + report.Result.Label, + report.Result.Iterations, + report.Result.TotalEvents, + report.Result.TotalRules, + report.Result.ActionsPerRule, + report.Result.AverageMatchesPerEvent, + report.Result.MinMatchesPerEvent, + report.Result.MaxMatchesPerEvent, + report.Result.AverageDeliveriesPerEvent, + report.Result.TotalDeliveries, + report.Result.MeanMs, + report.Result.P95Ms, + report.Result.MaxMs, + report.Result.MeanThroughputPerSecond, + report.Result.MinThroughputPerSecond, + report.Result.MaxAllocatedMb, + report.Result.ThresholdMs, + report.Result.MinThroughputThresholdPerSecond, + report.Result.MaxAllocatedThresholdMb, + baseline is null + ? null + : new BenchmarkJsonScenarioBaseline( + baseline.Iterations, + baseline.EventCount, + baseline.DeliveryCount, + baseline.MeanMs, + baseline.P95Ms, + baseline.MaxMs, + baseline.MeanThroughputPerSecond, + baseline.MinThroughputPerSecond, + baseline.MaxAllocatedMb), + new BenchmarkJsonScenarioRegression( + report.DurationRegressionRatio, + report.ThroughputRegressionRatio, + report.RegressionLimit, + report.RegressionBreached)); + } + + private sealed record BenchmarkJsonDocument( + string SchemaVersion, + DateTimeOffset CapturedAt, + string? Commit, + string? Environment, + IReadOnlyList Scenarios); + + private sealed record BenchmarkJsonScenario( + string Id, + string Label, + int Iterations, + int TotalEvents, + int TotalRules, + int ActionsPerRule, + double AverageMatchesPerEvent, + int MinMatchesPerEvent, + int MaxMatchesPerEvent, + double AverageDeliveriesPerEvent, + int TotalDeliveries, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb, + double? ThresholdMs, + double? MinThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb, + BenchmarkJsonScenarioBaseline? Baseline, + BenchmarkJsonScenarioRegression Regression); + + private sealed record BenchmarkJsonScenarioBaseline( + int Iterations, + int EventCount, + int DeliveryCount, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb); + + private sealed record BenchmarkJsonScenarioRegression( + double? DurationRatio, + double? ThroughputRatio, + double Limit, + bool Breached); +} + +internal sealed record BenchmarkJsonMetadata( + string SchemaVersion, + DateTimeOffset CapturedAtUtc, + string? Commit, + string? Environment); diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/BenchmarkScenarioReport.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/BenchmarkScenarioReport.cs new file mode 100644 index 00000000..f0a62903 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/BenchmarkScenarioReport.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using StellaOps.Bench.Notify.Baseline; + +namespace StellaOps.Bench.Notify.Reporting; + +internal sealed class BenchmarkScenarioReport +{ + private const double DefaultRegressionLimit = 1.15d; + + public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null) + { + Result = result ?? throw new ArgumentNullException(nameof(result)); + Baseline = baseline; + RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : DefaultRegressionLimit; + DurationRegressionRatio = CalculateDurationRatio(result.MaxMs, baseline?.MaxMs); + ThroughputRegressionRatio = CalculateThroughputRatio(result.MinThroughputPerSecond, baseline?.MinThroughputPerSecond); + } + + public ScenarioResult Result { get; } + + public BaselineEntry? Baseline { get; } + + public double RegressionLimit { get; } + + public double? DurationRegressionRatio { get; } + + public double? ThroughputRegressionRatio { get; } + + public bool DurationRegressionBreached => + DurationRegressionRatio is { } ratio && + ratio >= RegressionLimit; + + public bool ThroughputRegressionBreached => + ThroughputRegressionRatio is { } ratio && + ratio >= RegressionLimit; + + public bool RegressionBreached => DurationRegressionBreached || ThroughputRegressionBreached; + + public IEnumerable BuildRegressionFailureMessages() + { + if (Baseline is null) + { + yield break; + } + + if (DurationRegressionBreached && DurationRegressionRatio is { } durationRatio) + { + var delta = (durationRatio - 1d) * 100d; + yield return $"{Result.Id} exceeded max duration budget: {Result.MaxMs:F2} ms vs baseline {Baseline.MaxMs:F2} ms (+{delta:F1}%)."; + } + + if (ThroughputRegressionBreached && ThroughputRegressionRatio is { } throughputRatio) + { + var delta = (throughputRatio - 1d) * 100d; + yield return $"{Result.Id} throughput regressed: min {Result.MinThroughputPerSecond:N0} /s vs baseline {Baseline.MinThroughputPerSecond:N0} /s (-{delta:F1}%)."; + } + } + + private static double? CalculateDurationRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + return current / baseline.Value; + } + + private static double? CalculateThroughputRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + if (current <= 0d) + { + return double.PositiveInfinity; + } + + return baseline.Value / current; + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/PrometheusWriter.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/PrometheusWriter.cs new file mode 100644 index 00000000..d7c8f785 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/Reporting/PrometheusWriter.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace StellaOps.Bench.Notify.Reporting; + +internal static class PrometheusWriter +{ + public static void Write(string path, IReadOnlyList reports) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var builder = new StringBuilder(); + builder.AppendLine("# HELP notify_dispatch_bench_duration_ms Notify dispatch benchmark duration metrics (milliseconds)."); + builder.AppendLine("# TYPE notify_dispatch_bench_duration_ms gauge"); + builder.AppendLine("# HELP notify_dispatch_bench_throughput_per_sec Notify dispatch benchmark throughput metrics (deliveries per second)."); + builder.AppendLine("# TYPE notify_dispatch_bench_throughput_per_sec gauge"); + builder.AppendLine("# HELP notify_dispatch_bench_allocation_mb Notify dispatch benchmark allocation metrics (megabytes)."); + builder.AppendLine("# TYPE notify_dispatch_bench_allocation_mb gauge"); + + foreach (var report in reports) + { + var scenarioLabel = Escape(report.Result.Id); + AppendMetric(builder, "notify_dispatch_bench_mean_ms", scenarioLabel, report.Result.MeanMs); + AppendMetric(builder, "notify_dispatch_bench_p95_ms", scenarioLabel, report.Result.P95Ms); + AppendMetric(builder, "notify_dispatch_bench_max_ms", scenarioLabel, report.Result.MaxMs); + AppendMetric(builder, "notify_dispatch_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs); + + AppendMetric(builder, "notify_dispatch_bench_mean_throughput_per_sec", scenarioLabel, report.Result.MeanThroughputPerSecond); + AppendMetric(builder, "notify_dispatch_bench_min_throughput_per_sec", scenarioLabel, report.Result.MinThroughputPerSecond); + AppendMetric(builder, "notify_dispatch_bench_min_throughput_threshold_per_sec", scenarioLabel, report.Result.MinThroughputThresholdPerSecond); + + AppendMetric(builder, "notify_dispatch_bench_max_allocated_mb", scenarioLabel, report.Result.MaxAllocatedMb); + AppendMetric(builder, "notify_dispatch_bench_max_allocated_threshold_mb", scenarioLabel, report.Result.MaxAllocatedThresholdMb); + + if (report.Baseline is { } baseline) + { + AppendMetric(builder, "notify_dispatch_bench_baseline_max_ms", scenarioLabel, baseline.MaxMs); + AppendMetric(builder, "notify_dispatch_bench_baseline_mean_ms", scenarioLabel, baseline.MeanMs); + AppendMetric(builder, "notify_dispatch_bench_baseline_min_throughput_per_sec", scenarioLabel, baseline.MinThroughputPerSecond); + } + + if (report.DurationRegressionRatio is { } durationRatio) + { + AppendMetric(builder, "notify_dispatch_bench_duration_regression_ratio", scenarioLabel, durationRatio); + } + + if (report.ThroughputRegressionRatio is { } throughputRatio) + { + AppendMetric(builder, "notify_dispatch_bench_throughput_regression_ratio", scenarioLabel, throughputRatio); + } + + AppendMetric(builder, "notify_dispatch_bench_regression_limit", scenarioLabel, report.RegressionLimit); + AppendMetric(builder, "notify_dispatch_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0); + } + + File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8); + } + + private static void AppendMetric(StringBuilder builder, string metric, string scenario, double? value) + { + if (!value.HasValue) + { + return; + } + + builder.Append(metric); + builder.Append("{scenario=\""); + builder.Append(scenario); + builder.Append("\"} "); + builder.AppendLine(value.Value.ToString("G17", CultureInfo.InvariantCulture)); + } + + private static string Escape(string value) => + value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioExecutionResult.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioExecutionResult.cs new file mode 100644 index 00000000..f205ec88 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioExecutionResult.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; + +namespace StellaOps.Bench.Notify; + +internal sealed record ScenarioExecutionResult( + IReadOnlyList Durations, + IReadOnlyList Throughputs, + IReadOnlyList AllocatedMb, + int TotalEvents, + int TotalRules, + int ActionsPerRule, + double AverageMatchesPerEvent, + int MinMatchesPerEvent, + int MaxMatchesPerEvent, + double AverageDeliveriesPerEvent, + int TotalMatches, + int TotalDeliveries); diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioResult.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioResult.cs new file mode 100644 index 00000000..8574ed8d --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioResult.cs @@ -0,0 +1,46 @@ +using System.Globalization; + +namespace StellaOps.Bench.Notify; + +internal sealed record ScenarioResult( + string Id, + string Label, + int Iterations, + int TotalEvents, + int TotalRules, + int ActionsPerRule, + double AverageMatchesPerEvent, + int MinMatchesPerEvent, + int MaxMatchesPerEvent, + double AverageDeliveriesPerEvent, + int TotalDeliveries, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb, + double? ThresholdMs, + double? MinThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb) +{ + public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28]; + + public string EventsColumn => TotalEvents.ToString("N0", CultureInfo.InvariantCulture).PadLeft(12); + + public string RulesColumn => TotalRules.ToString("N0", CultureInfo.InvariantCulture).PadLeft(9); + + public string MatchesColumn => AverageMatchesPerEvent.ToString("F1", CultureInfo.InvariantCulture).PadLeft(8); + + public string DeliveriesColumn => AverageDeliveriesPerEvent.ToString("F1", CultureInfo.InvariantCulture).PadLeft(10); + + public string MeanColumn => MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string P95Column => P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string MaxColumn => MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + + public string MinThroughputColumn => (MinThroughputPerSecond / 1_000d).ToString("F2", CultureInfo.InvariantCulture).PadLeft(11); + + public string AllocatedColumn => MaxAllocatedMb.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9); +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioStatistics.cs b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioStatistics.cs new file mode 100644 index 00000000..b15ed639 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/ScenarioStatistics.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Bench.Notify; + +internal readonly record struct DurationStatistics(double MeanMs, double P95Ms, double MaxMs) +{ + public static DurationStatistics From(IReadOnlyList durations) + { + if (durations.Count == 0) + { + return new DurationStatistics(0, 0, 0); + } + + var sorted = durations.ToArray(); + Array.Sort(sorted); + + var total = 0d; + foreach (var value in durations) + { + total += value; + } + + var mean = total / durations.Count; + var p95 = Percentile(sorted, 95); + var max = sorted[^1]; + + return new DurationStatistics(mean, p95, max); + } + + private static double Percentile(IReadOnlyList sorted, double percentile) + { + if (sorted.Count == 0) + { + return 0; + } + + var rank = (percentile / 100d) * (sorted.Count - 1); + var lower = (int)Math.Floor(rank); + var upper = (int)Math.Ceiling(rank); + var weight = rank - lower; + + if (upper >= sorted.Count) + { + return sorted[lower]; + } + + return sorted[lower] + weight * (sorted[upper] - sorted[lower]); + } +} + +internal readonly record struct ThroughputStatistics(double MeanPerSecond, double MinPerSecond) +{ + public static ThroughputStatistics From(IReadOnlyList values) + { + if (values.Count == 0) + { + return new ThroughputStatistics(0, 0); + } + + var total = 0d; + var min = double.MaxValue; + + foreach (var value in values) + { + total += value; + min = Math.Min(min, value); + } + + var mean = total / values.Count; + return new ThroughputStatistics(mean, min); + } +} + +internal readonly record struct AllocationStatistics(double MaxAllocatedMb) +{ + public static AllocationStatistics From(IReadOnlyList values) + { + var max = 0d; + foreach (var value in values) + { + max = Math.Max(max, value); + } + + return new AllocationStatistics(max); + } +} diff --git a/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/StellaOps.Bench.Notify.csproj b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/StellaOps.Bench.Notify.csproj new file mode 100644 index 00000000..55c3f3a2 --- /dev/null +++ b/src/StellaOps.Bench/Notify/StellaOps.Bench.Notify/StellaOps.Bench.Notify.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + preview + true + + + + + + diff --git a/src/StellaOps.Bench/Notify/baseline.csv b/src/StellaOps.Bench/Notify/baseline.csv new file mode 100644 index 00000000..d79c2c45 --- /dev/null +++ b/src/StellaOps.Bench/Notify/baseline.csv @@ -0,0 +1,4 @@ +scenario,iterations,events,deliveries,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb +notify_dispatch_density_05,5,5000,20000,3.4150,4.1722,4.3039,6053938.5172,4646948.1168,0.0000 +notify_dispatch_density_20,5,7500,675000,24.2274,25.8517,26.0526,27923335.5855,25909122.3141,0.0000 +notify_dispatch_density_40,5,10000,4000080,138.7387,147.7174,149.1124,28916602.9214,26825938.0172,0.0000 diff --git a/src/StellaOps.Bench/Notify/config.json b/src/StellaOps.Bench/Notify/config.json new file mode 100644 index 00000000..ecd9c5b5 --- /dev/null +++ b/src/StellaOps.Bench/Notify/config.json @@ -0,0 +1,47 @@ +{ + "thresholdMs": 1200, + "minThroughputPerSecond": 10000, + "maxAllocatedMb": 512, + "iterations": 5, + "scenarios": [ + { + "id": "notify_dispatch_density_05", + "label": "50 rules / 5% fanout", + "eventCount": 5000, + "ruleCount": 50, + "actionsPerRule": 2, + "matchRate": 0.05, + "tenantCount": 4, + "channelCount": 12, + "thresholdMs": 400, + "minThroughputPerSecond": 15000, + "maxAllocatedMb": 128 + }, + { + "id": "notify_dispatch_density_20", + "label": "150 rules / 20% fanout", + "eventCount": 7500, + "ruleCount": 150, + "actionsPerRule": 3, + "matchRate": 0.2, + "tenantCount": 6, + "channelCount": 24, + "thresholdMs": 650, + "minThroughputPerSecond": 30000, + "maxAllocatedMb": 192 + }, + { + "id": "notify_dispatch_density_40", + "label": "300 rules / 40% fanout", + "eventCount": 10000, + "ruleCount": 300, + "actionsPerRule": 4, + "matchRate": 0.4, + "tenantCount": 8, + "channelCount": 32, + "thresholdMs": 900, + "minThroughputPerSecond": 45000, + "maxAllocatedMb": 256 + } + ] +} diff --git a/src/StellaOps.Bench/PolicyEngine/README.md b/src/StellaOps.Bench/PolicyEngine/README.md new file mode 100644 index 00000000..9dac29d2 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/README.md @@ -0,0 +1,24 @@ +# Policy Engine Bench + +Synthetic workload that measures end-to-end policy evaluation throughput against large advisory snapshots. + +## Scenarios + +`config.json` defines the default scenario (`policy_eval_baseline`) which generates **100 000 components** with **1 000 000 findings** derived from deterministic seeds. The harness measures: + +- Latency (mean/p95/max milliseconds) +- Throughput (findings per second) +- Managed heap allocations (megabytes) + +## Running locally + +```bash +dotnet run \ + --project src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/StellaOps.Bench.PolicyEngine.csproj \ + -- \ + --csv out/policy-bench.csv \ + --json out/policy-bench.json \ + --prometheus out/policy-bench.prom +``` + +The run exits non-zero if latency exceeds configured thresholds, throughput falls below the configured floor, or allocations surpass the budget. Baselines live in `baseline.csv` and can be regenerated whenever new steady-state metrics are established. diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Baseline/BaselineEntry.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Baseline/BaselineEntry.cs new file mode 100644 index 00000000..8308f42a --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Baseline/BaselineEntry.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Bench.PolicyEngine.Baseline; + +internal sealed record BaselineEntry( + string ScenarioId, + int Iterations, + int FindingCount, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb); diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Baseline/BaselineLoader.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Baseline/BaselineLoader.cs new file mode 100644 index 00000000..d5bd4a99 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Baseline/BaselineLoader.cs @@ -0,0 +1,86 @@ +using System.Globalization; + +namespace StellaOps.Bench.PolicyEngine.Baseline; + +internal static class BaselineLoader +{ + public static async Task> LoadAsync(string path, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read); + using var reader = new StreamReader(stream); + + var lineNumber = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var line = await reader.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + lineNumber++; + if (lineNumber == 1) + { + continue; // header + } + + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = line.Split(',', StringSplitOptions.TrimEntries); + if (parts.Length < 9) + { + throw new InvalidOperationException($"Baseline '{resolved}' line {lineNumber} is invalid (expected 9 columns, found {parts.Length})."); + } + + var entry = new BaselineEntry( + ScenarioId: parts[0], + Iterations: ParseInt(parts[1], resolved, lineNumber), + FindingCount: ParseInt(parts[2], resolved, lineNumber), + MeanMs: ParseDouble(parts[3], resolved, lineNumber), + P95Ms: ParseDouble(parts[4], resolved, lineNumber), + MaxMs: ParseDouble(parts[5], resolved, lineNumber), + MeanThroughputPerSecond: ParseDouble(parts[6], resolved, lineNumber), + MinThroughputPerSecond: ParseDouble(parts[7], resolved, lineNumber), + MaxAllocatedMb: ParseDouble(parts[8], resolved, lineNumber)); + + result[entry.ScenarioId] = entry; + } + + return result; + } + + private static int ParseInt(string value, string file, int line) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var result)) + { + return result; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid integer '{value}'."); + } + + private static double ParseDouble(string value, string file, int line) + { + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var result)) + { + return result; + } + + throw new InvalidOperationException($"Baseline '{file}' line {line} contains an invalid number '{value}'."); + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/BenchmarkConfig.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/BenchmarkConfig.cs new file mode 100644 index 00000000..a325b784 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/BenchmarkConfig.cs @@ -0,0 +1,155 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Bench.PolicyEngine; + +internal sealed record BenchmarkConfig( + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MaxAllocatedMb, + int? Iterations, + IReadOnlyList Scenarios) +{ + public static async Task LoadAsync(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + throw new FileNotFoundException($"Benchmark configuration '{resolved}' was not found.", resolved); + } + + await using var stream = File.OpenRead(resolved); + var model = await JsonSerializer.DeserializeAsync( + stream, + new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }).ConfigureAwait(false); + + if (model is null) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' could not be parsed."); + } + + if (model.Scenarios.Count == 0) + { + throw new InvalidOperationException($"Benchmark configuration '{resolved}' does not contain any scenarios."); + } + + foreach (var scenario in model.Scenarios) + { + scenario.Validate(); + } + + return new BenchmarkConfig( + model.ThresholdMs, + model.MinThroughputPerSecond, + model.MaxAllocatedMb, + model.Iterations, + model.Scenarios); + } + + private sealed class BenchmarkConfigModel + { + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + [JsonPropertyName("iterations")] + public int? Iterations { get; init; } + + [JsonPropertyName("scenarios")] + public List Scenarios { get; init; } = new(); + } +} + +internal sealed class PolicyScenarioConfig +{ + private const int DefaultComponentCount = 100_000; + private const int DefaultAdvisoriesPerComponent = 10; + + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } + + [JsonPropertyName("policyPath")] + public string PolicyPath { get; init; } = "docs/examples/policies/baseline.yaml"; + + [JsonPropertyName("scoringConfig")] + public string? ScoringConfigPath { get; init; } + + [JsonPropertyName("componentCount")] + public int ComponentCount { get; init; } = DefaultComponentCount; + + [JsonPropertyName("advisoriesPerComponent")] + public int AdvisoriesPerComponent { get; init; } = DefaultAdvisoriesPerComponent; + + [JsonPropertyName("totalFindings")] + public int? TotalFindings { get; init; } + + [JsonPropertyName("seed")] + public int? Seed { get; init; } + + [JsonPropertyName("thresholdMs")] + public double? ThresholdMs { get; init; } + + [JsonPropertyName("minThroughputPerSecond")] + public double? MinThroughputPerSecond { get; init; } + + [JsonPropertyName("maxAllocatedMb")] + public double? MaxAllocatedMb { get; init; } + + public string ScenarioId => string.IsNullOrWhiteSpace(Id) ? "policy_eval" : Id.Trim(); + + public int ResolveFindingCount() + { + if (TotalFindings is { } findings) + { + if (findings <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires totalFindings > 0."); + } + + return findings; + } + + if (ComponentCount <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires componentCount > 0."); + } + + if (AdvisoriesPerComponent <= 0) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires advisoriesPerComponent > 0."); + } + + checked + { + var total = ComponentCount * AdvisoriesPerComponent; + return total; + } + } + + public int ResolveSeed() => Seed ?? 2025_10_26; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(PolicyPath)) + { + throw new InvalidOperationException($"Scenario '{ScenarioId}' requires a policyPath."); + } + + ResolveFindingCount(); + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PathUtilities.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PathUtilities.cs new file mode 100644 index 00000000..c16b67af --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PathUtilities.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Bench.PolicyEngine; + +internal static class PathUtilities +{ + public static bool IsWithinRoot(string root, string candidate) + { + var relative = Path.GetRelativePath(root, candidate); + if (string.IsNullOrEmpty(relative) || relative == ".") + { + return true; + } + + return !relative.StartsWith("..", StringComparison.Ordinal) && !Path.IsPathRooted(relative); + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs new file mode 100644 index 00000000..16afe6a6 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs @@ -0,0 +1,249 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using StellaOps.Policy; + +namespace StellaOps.Bench.PolicyEngine; + +internal sealed class PolicyScenarioRunner +{ + private readonly PolicyScenarioConfig _config; + private readonly PolicyDocument _document; + private readonly PolicyScoringConfig _scoringConfig; + private readonly PolicyFinding[] _findings; + + public PolicyScenarioRunner(PolicyScenarioConfig config, string repoRoot) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot); + + var policyPath = ResolvePathWithinRoot(repoRoot, config.PolicyPath); + var policyContent = File.ReadAllText(policyPath); + var policyFormat = PolicySchema.DetectFormat(policyPath); + var binding = PolicyBinder.Bind(policyContent, policyFormat); + if (!binding.Success) + { + var issues = string.Join(", ", binding.Issues.Select(issue => issue.Code)); + throw new InvalidOperationException($"Policy '{config.PolicyPath}' failed validation: {issues}."); + } + + _document = binding.Document; + + _scoringConfig = LoadScoringConfig(repoRoot, config.ScoringConfigPath); + _findings = SyntheticFindingGenerator.Create(config, repoRoot); + } + + public ScenarioExecutionResult Execute(int iterations, CancellationToken cancellationToken) + { + if (iterations <= 0) + { + throw new ArgumentOutOfRangeException(nameof(iterations), iterations, "Iterations must be positive."); + } + + var durations = new double[iterations]; + var throughputs = new double[iterations]; + var allocations = new double[iterations]; + var hashingAccumulator = new EvaluationAccumulator(); + + for (var index = 0; index < iterations; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + + var beforeAllocated = GC.GetTotalAllocatedBytes(); + var stopwatch = Stopwatch.StartNew(); + + hashingAccumulator.Reset(); + foreach (var finding in _findings) + { + var verdict = PolicyEvaluation.EvaluateFinding(_document, _scoringConfig, finding); + hashingAccumulator.Add(verdict); + } + + stopwatch.Stop(); + + var afterAllocated = GC.GetTotalAllocatedBytes(); + var elapsedMs = stopwatch.Elapsed.TotalMilliseconds; + if (elapsedMs <= 0) + { + elapsedMs = 0.0001; + } + + durations[index] = elapsedMs; + throughputs[index] = _findings.Length / stopwatch.Elapsed.TotalSeconds; + allocations[index] = Math.Max(0, afterAllocated - beforeAllocated) / (1024d * 1024d); + + hashingAccumulator.AssertConsumed(); + } + + return new ScenarioExecutionResult( + durations, + throughputs, + allocations, + _findings.Length); + } + + private static PolicyScoringConfig LoadScoringConfig(string repoRoot, string? scoringPath) + { + if (string.IsNullOrWhiteSpace(scoringPath)) + { + return PolicyScoringConfig.Default; + } + + var resolved = ResolvePathWithinRoot(repoRoot, scoringPath); + var format = PolicySchema.DetectFormat(resolved); + var content = File.ReadAllText(resolved); + var binding = PolicyScoringConfigBinder.Bind(content, format); + if (!binding.Success || binding.Config is null) + { + var issues = binding.Issues.Length == 0 + ? "unknown" + : string.Join(", ", binding.Issues.Select(issue => issue.Code)); + throw new InvalidOperationException($"Scoring configuration '{scoringPath}' failed validation: {issues}."); + } + + return binding.Config; + } + + private static string ResolvePathWithinRoot(string repoRoot, string relativePath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(repoRoot); + ArgumentException.ThrowIfNullOrWhiteSpace(relativePath); + + var combined = Path.GetFullPath(Path.Combine(repoRoot, relativePath)); + if (!PathUtilities.IsWithinRoot(repoRoot, combined)) + { + throw new InvalidOperationException($"Path '{relativePath}' escapes repository root '{repoRoot}'."); + } + + if (!File.Exists(combined)) + { + throw new FileNotFoundException($"Path '{relativePath}' resolved to '{combined}' but does not exist.", combined); + } + + return combined; + } +} + +internal sealed record ScenarioExecutionResult( + IReadOnlyList Durations, + IReadOnlyList Throughputs, + IReadOnlyList AllocatedMb, + int FindingCount); + +internal static class SyntheticFindingGenerator +{ + private static readonly ImmutableArray Environments = ImmutableArray.Create("prod", "staging", "dev"); + private static readonly ImmutableArray Sources = ImmutableArray.Create("concelier", "excitor", "sbom"); + private static readonly ImmutableArray Vendors = ImmutableArray.Create("acme", "contoso", "globex", "initech", "umbrella"); + private static readonly ImmutableArray Licenses = ImmutableArray.Create("MIT", "Apache-2.0", "GPL-3.0", "BSD-3-Clause", "Proprietary"); + private static readonly ImmutableArray Repositories = ImmutableArray.Create("acme/service-api", "acme/web", "acme/worker", "acme/mobile", "acme/cli"); + private static readonly ImmutableArray Images = ImmutableArray.Create("registry.local/worker:2025.10", "registry.local/api:2025.10", "registry.local/cli:2025.10"); + private static readonly ImmutableArray TagPool = ImmutableArray.Create("kev", "runtime", "reachable", "public", "third-party", "critical-path"); + private static readonly ImmutableArray> TagSets = BuildTagSets(); + private static readonly PolicySeverity[] SeverityPool = + { + PolicySeverity.Critical, + PolicySeverity.High, + PolicySeverity.Medium, + PolicySeverity.Low, + PolicySeverity.Informational + }; + + public static PolicyFinding[] Create(PolicyScenarioConfig config, string repoRoot) + { + var totalFindings = config.ResolveFindingCount(); + if (totalFindings <= 0) + { + return Array.Empty(); + } + + var seed = config.ResolveSeed(); + var random = new Random(seed); + var findings = new PolicyFinding[totalFindings]; + var tagsBuffer = new List(3); + + var componentCount = Math.Max(1, config.ComponentCount); + + for (var index = 0; index < totalFindings; index++) + { + var componentIndex = index % componentCount; + var findingId = $"F-{componentIndex:D5}-{index:D6}"; + var severity = SeverityPool[random.Next(SeverityPool.Length)]; + var environment = Environments[componentIndex % Environments.Length]; + var source = Sources[random.Next(Sources.Length)]; + var vendor = Vendors[random.Next(Vendors.Length)]; + var license = Licenses[random.Next(Licenses.Length)]; + var repository = Repositories[componentIndex % Repositories.Length]; + var image = Images[(componentIndex + index) % Images.Length]; + var packageName = $"pkg{componentIndex % 1000}"; + var purl = $"pkg:generic/{packageName}@{1 + (index % 20)}.{1 + (componentIndex % 10)}.{index % 5}"; + var cve = index % 7 == 0 ? $"CVE-2025-{1000 + index % 9000:D4}" : null; + var layerDigest = $"sha256:{Convert.ToHexString(Guid.NewGuid().ToByteArray())[..32].ToLowerInvariant()}"; + + var tags = TagSets[random.Next(TagSets.Length)]; + + findings[index] = PolicyFinding.Create( + findingId, + severity, + environment: environment, + source: source, + vendor: vendor, + license: license, + image: image, + repository: repository, + package: packageName, + purl: purl, + cve: cve, + path: $"/app/{packageName}/{index % 50}.so", + layerDigest: layerDigest, + tags: tags); + } + + return findings; + } + + private static ImmutableArray> BuildTagSets() + { + var builder = ImmutableArray.CreateBuilder>(); + builder.Add(ImmutableArray.Empty); + builder.Add(ImmutableArray.Create("kev")); + builder.Add(ImmutableArray.Create("runtime")); + builder.Add(ImmutableArray.Create("reachable")); + builder.Add(ImmutableArray.Create("third-party")); + builder.Add(ImmutableArray.Create("kev", "runtime")); + builder.Add(ImmutableArray.Create("kev", "third-party")); + builder.Add(ImmutableArray.Create("runtime", "public")); + builder.Add(ImmutableArray.Create("reachable", "critical-path")); + return builder.ToImmutable(); + } +} + +internal sealed class EvaluationAccumulator +{ + private double _scoreAccumulator; + private int _quietCount; + + public void Reset() + { + _scoreAccumulator = 0; + _quietCount = 0; + } + + public void Add(PolicyVerdict verdict) + { + _scoreAccumulator += verdict.Score; + if (verdict.Quiet) + { + _quietCount++; + } + } + + public void AssertConsumed() + { + if (_scoreAccumulator == 0 && _quietCount == 0) + { + throw new InvalidOperationException("Evaluation accumulator detected zero work; dataset may be empty."); + } + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Program.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Program.cs new file mode 100644 index 00000000..6ca61d97 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Program.cs @@ -0,0 +1,373 @@ +using System.Globalization; +using System.Linq; +using StellaOps.Bench.PolicyEngine.Baseline; +using StellaOps.Bench.PolicyEngine.Reporting; + +namespace StellaOps.Bench.PolicyEngine; + +internal static class Program +{ + public static async Task Main(string[] args) + { + try + { + var options = ProgramOptions.Parse(args); + var config = await BenchmarkConfig.LoadAsync(options.ConfigPath).ConfigureAwait(false); + var iterations = options.Iterations ?? config.Iterations ?? 3; + var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath); + var thresholdMs = options.ThresholdMs ?? config.ThresholdMs; + var throughputFloor = options.MinThroughputPerSecond ?? config.MinThroughputPerSecond; + var allocationLimit = options.MaxAllocatedMb ?? config.MaxAllocatedMb; + var regressionLimit = options.RegressionLimit; + var capturedAt = (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime(); + + var baseline = await BaselineLoader.LoadAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false); + + var results = new List(); + var reports = new List(); + var failures = new List(); + + foreach (var scenario in config.Scenarios) + { + var runner = new PolicyScenarioRunner(scenario, repoRoot); + var execution = runner.Execute(iterations, CancellationToken.None); + + var durationStats = DurationStatistics.From(execution.Durations); + var throughputStats = ThroughputStatistics.From(execution.Throughputs); + var allocationStats = AllocationStatistics.From(execution.AllocatedMb); + + var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs; + var scenarioThroughputFloor = scenario.MinThroughputPerSecond ?? throughputFloor; + var scenarioAllocationLimit = scenario.MaxAllocatedMb ?? allocationLimit; + + var result = new ScenarioResult( + scenario.ScenarioId, + scenario.Label ?? scenario.ScenarioId, + iterations, + execution.FindingCount, + durationStats.MeanMs, + durationStats.P95Ms, + durationStats.MaxMs, + throughputStats.MeanPerSecond, + throughputStats.MinPerSecond, + allocationStats.MaxAllocatedMb, + scenarioThreshold, + scenarioThroughputFloor, + scenarioAllocationLimit); + + results.Add(result); + + if (scenarioThreshold is { } threshold && result.MaxMs > threshold) + { + failures.Add($"{result.Id} exceeded latency threshold: {result.MaxMs:F2} ms > {threshold:F2} ms"); + } + + if (scenarioThroughputFloor is { } floor && result.MinThroughputPerSecond < floor) + { + failures.Add($"{result.Id} fell below throughput floor: {result.MinThroughputPerSecond:N0} findings/s < {floor:N0} findings/s"); + } + + if (scenarioAllocationLimit is { } limit && result.MaxAllocatedMb > limit) + { + failures.Add($"{result.Id} exceeded allocation budget: {result.MaxAllocatedMb:F2} MB > {limit:F2} MB"); + } + + baseline.TryGetValue(result.Id, out var baselineEntry); + var report = new BenchmarkScenarioReport(result, baselineEntry, regressionLimit); + reports.Add(report); + failures.AddRange(report.BuildRegressionFailureMessages()); + } + + TablePrinter.Print(results); + + if (!string.IsNullOrWhiteSpace(options.CsvOutPath)) + { + CsvWriter.Write(options.CsvOutPath!, results); + } + + if (!string.IsNullOrWhiteSpace(options.JsonOutPath)) + { + var metadata = new BenchmarkJsonMetadata( + SchemaVersion: "policy-bench/1.0", + CapturedAtUtc: capturedAt, + Commit: options.Commit, + Environment: options.Environment); + + await BenchmarkJsonWriter.WriteAsync( + options.JsonOutPath!, + metadata, + reports, + CancellationToken.None).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath)) + { + PrometheusWriter.Write(options.PrometheusOutPath!, reports); + } + + if (failures.Count > 0) + { + Console.Error.WriteLine(); + Console.Error.WriteLine("Benchmark failures detected:"); + foreach (var failure in failures.Distinct()) + { + Console.Error.WriteLine($" - {failure}"); + } + + return 1; + } + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"policy-bench error: {ex.Message}"); + return 1; + } + } + + private static string ResolveRepoRoot(string? overridePath, string configPath) + { + if (!string.IsNullOrWhiteSpace(overridePath)) + { + return Path.GetFullPath(overridePath); + } + + var configDirectory = Path.GetDirectoryName(configPath); + if (string.IsNullOrWhiteSpace(configDirectory)) + { + return Directory.GetCurrentDirectory(); + } + + return Path.GetFullPath(Path.Combine(configDirectory, "..", "..", "..")); + } + + private sealed record ProgramOptions( + string ConfigPath, + int? Iterations, + double? ThresholdMs, + double? MinThroughputPerSecond, + double? MaxAllocatedMb, + string? CsvOutPath, + string? JsonOutPath, + string? PrometheusOutPath, + string? RepoRoot, + string BaselinePath, + DateTimeOffset? CapturedAtUtc, + string? Commit, + string? Environment, + double? RegressionLimit) + { + public static ProgramOptions Parse(string[] args) + { + var configPath = DefaultConfigPath(); + var baselinePath = DefaultBaselinePath(); + + int? iterations = null; + double? thresholdMs = null; + double? minThroughput = null; + double? maxAllocated = null; + string? csvOut = null; + string? jsonOut = null; + string? promOut = null; + string? repoRoot = null; + DateTimeOffset? capturedAt = null; + string? commit = null; + string? environment = null; + double? regressionLimit = null; + + for (var index = 0; index < args.Length; index++) + { + var current = args[index]; + switch (current) + { + case "--config": + EnsureNext(args, index); + configPath = Path.GetFullPath(args[++index]); + break; + case "--iterations": + EnsureNext(args, index); + iterations = int.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--threshold-ms": + EnsureNext(args, index); + thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--min-throughput": + EnsureNext(args, index); + minThroughput = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--max-allocated-mb": + EnsureNext(args, index); + maxAllocated = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--csv": + EnsureNext(args, index); + csvOut = args[++index]; + break; + case "--json": + EnsureNext(args, index); + jsonOut = args[++index]; + break; + case "--prometheus": + EnsureNext(args, index); + promOut = args[++index]; + break; + case "--repo-root": + EnsureNext(args, index); + repoRoot = args[++index]; + break; + case "--baseline": + EnsureNext(args, index); + baselinePath = Path.GetFullPath(args[++index]); + break; + case "--captured-at": + EnsureNext(args, index); + capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal); + break; + case "--commit": + EnsureNext(args, index); + commit = args[++index]; + break; + case "--environment": + EnsureNext(args, index); + environment = args[++index]; + break; + case "--regression-limit": + EnsureNext(args, index); + regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; + case "--help": + case "-h": + PrintUsage(); + System.Environment.Exit(0); + break; + default: + throw new ArgumentException($"Unknown argument '{current}'."); + } + } + + return new ProgramOptions( + configPath, + iterations, + thresholdMs, + minThroughput, + maxAllocated, + csvOut, + jsonOut, + promOut, + repoRoot, + baselinePath, + capturedAt, + commit, + environment, + regressionLimit); + } + + private static string DefaultConfigPath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "config.json"); + } + + private static string DefaultBaselinePath() + { + var binaryDir = AppContext.BaseDirectory; + var projectDir = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectDir, "..")); + return Path.Combine(benchRoot, "baseline.csv"); + } + + private static void EnsureNext(string[] args, int index) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException("Missing value for argument."); + } + } + + private static void PrintUsage() + { + Console.WriteLine("Usage: policy-bench [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --config Path to benchmark configuration JSON."); + Console.WriteLine(" --iterations Override iteration count."); + Console.WriteLine(" --threshold-ms Global latency threshold in milliseconds."); + Console.WriteLine(" --min-throughput Global throughput floor (findings/second)."); + Console.WriteLine(" --max-allocated-mb Global allocation ceiling (MB)."); + Console.WriteLine(" --csv Write CSV results to path."); + Console.WriteLine(" --json Write JSON results to path."); + Console.WriteLine(" --prometheus Write Prometheus exposition metrics to path."); + Console.WriteLine(" --repo-root Repository root override."); + Console.WriteLine(" --baseline Baseline CSV path."); + Console.WriteLine(" --captured-at Timestamp to embed in JSON metadata."); + Console.WriteLine(" --commit Commit identifier for metadata."); + Console.WriteLine(" --environment Environment label for metadata."); + Console.WriteLine(" --regression-limit Regression multiplier (default 1.15)."); + } + } +} + +internal static class TablePrinter +{ + public static void Print(IEnumerable results) + { + Console.WriteLine("Scenario | Findings | Mean(ms) | P95(ms) | Max(ms) | Min k/s | Alloc(MB)"); + Console.WriteLine("---------------------------- | ----------- | ---------- | ---------- | ---------- | -------- | --------"); + foreach (var row in results) + { + Console.WriteLine(string.Join(" | ", new[] + { + row.IdColumn, + row.FindingsColumn, + row.MeanColumn, + row.P95Column, + row.MaxColumn, + row.MinThroughputColumn, + row.AllocatedColumn + })); + } + } +} + +internal static class CsvWriter +{ + public static void Write(string path, IEnumerable results) + { + var resolvedPath = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolvedPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + using var stream = new FileStream(resolvedPath, FileMode.Create, FileAccess.Write, FileShare.None); + using var writer = new StreamWriter(stream); + writer.WriteLine("scenario,iterations,findings,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb"); + + foreach (var row in results) + { + writer.Write(row.Id); + writer.Write(','); + writer.Write(row.Iterations.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.FindingCount.ToString(CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MeanMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.P95Ms.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MaxMs.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MeanThroughputPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MinThroughputPerSecond.ToString("F4", CultureInfo.InvariantCulture)); + writer.Write(','); + writer.Write(row.MaxAllocatedMb.ToString("F4", CultureInfo.InvariantCulture)); + writer.WriteLine(); + } + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/BenchmarkJsonWriter.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/BenchmarkJsonWriter.cs new file mode 100644 index 00000000..16cf6299 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/BenchmarkJsonWriter.cs @@ -0,0 +1,125 @@ +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Bench.PolicyEngine.Baseline; + +namespace StellaOps.Bench.PolicyEngine.Reporting; + +internal static class BenchmarkJsonWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + public static async Task WriteAsync( + string path, + BenchmarkJsonMetadata metadata, + IReadOnlyList reports, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(metadata); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var document = new BenchmarkJsonDocument( + metadata.SchemaVersion, + metadata.CapturedAtUtc, + metadata.Commit, + metadata.Environment, + reports.Select(CreateScenario).ToArray()); + + await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report) + { + var baseline = report.Baseline; + return new BenchmarkJsonScenario( + report.Result.Id, + report.Result.Label, + report.Result.Iterations, + report.Result.FindingCount, + report.Result.MeanMs, + report.Result.P95Ms, + report.Result.MaxMs, + report.Result.MeanThroughputPerSecond, + report.Result.MinThroughputPerSecond, + report.Result.MaxAllocatedMb, + report.Result.ThresholdMs, + report.Result.MinThroughputThresholdPerSecond, + report.Result.MaxAllocatedThresholdMb, + baseline is null + ? null + : new BenchmarkJsonScenarioBaseline( + baseline.Iterations, + baseline.FindingCount, + baseline.MeanMs, + baseline.P95Ms, + baseline.MaxMs, + baseline.MeanThroughputPerSecond, + baseline.MinThroughputPerSecond, + baseline.MaxAllocatedMb), + new BenchmarkJsonScenarioRegression( + report.DurationRegressionRatio, + report.ThroughputRegressionRatio, + report.RegressionLimit, + report.RegressionBreached)); + } + + private sealed record BenchmarkJsonDocument( + string SchemaVersion, + DateTimeOffset CapturedAt, + string? Commit, + string? Environment, + IReadOnlyList Scenarios); + + private sealed record BenchmarkJsonScenario( + string Id, + string Label, + int Iterations, + int FindingCount, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb, + double? ThresholdMs, + double? MinThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb, + BenchmarkJsonScenarioBaseline? Baseline, + BenchmarkJsonScenarioRegression Regression); + + private sealed record BenchmarkJsonScenarioBaseline( + int Iterations, + int FindingCount, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb); + + private sealed record BenchmarkJsonScenarioRegression( + double? DurationRatio, + double? ThroughputRatio, + double Limit, + bool Breached); +} + +internal sealed record BenchmarkJsonMetadata( + string SchemaVersion, + DateTimeOffset CapturedAtUtc, + string? Commit, + string? Environment); diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/BenchmarkScenarioReport.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/BenchmarkScenarioReport.cs new file mode 100644 index 00000000..5a705a71 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/BenchmarkScenarioReport.cs @@ -0,0 +1,82 @@ +using StellaOps.Bench.PolicyEngine.Baseline; + +namespace StellaOps.Bench.PolicyEngine.Reporting; + +internal sealed class BenchmarkScenarioReport +{ + private const double DefaultRegressionLimit = 1.15d; + + public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null) + { + Result = result ?? throw new ArgumentNullException(nameof(result)); + Baseline = baseline; + RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : DefaultRegressionLimit; + DurationRegressionRatio = CalculateDurationRatio(result.MaxMs, baseline?.MaxMs); + ThroughputRegressionRatio = CalculateThroughputRatio(result.MinThroughputPerSecond, baseline?.MinThroughputPerSecond); + } + + public ScenarioResult Result { get; } + + public BaselineEntry? Baseline { get; } + + public double RegressionLimit { get; } + + public double? DurationRegressionRatio { get; } + + public double? ThroughputRegressionRatio { get; } + + public bool DurationRegressionBreached => + DurationRegressionRatio is { } ratio && + ratio >= RegressionLimit; + + public bool ThroughputRegressionBreached => + ThroughputRegressionRatio is { } ratio && + ratio >= RegressionLimit; + + public bool RegressionBreached => DurationRegressionBreached || ThroughputRegressionBreached; + + public IEnumerable BuildRegressionFailureMessages() + { + if (Baseline is null) + { + yield break; + } + + if (DurationRegressionBreached && DurationRegressionRatio is { } durationRatio) + { + var delta = (durationRatio - 1d) * 100d; + yield return $"{Result.Id} exceeded max duration budget: {Result.MaxMs:F2} ms vs baseline {Baseline.MaxMs:F2} ms (+{delta:F1}%)."; + } + + if (ThroughputRegressionBreached && ThroughputRegressionRatio is { } throughputRatio) + { + var delta = (throughputRatio - 1d) * 100d; + yield return $"{Result.Id} throughput regressed: min {Result.MinThroughputPerSecond:N0} /s vs baseline {Baseline.MinThroughputPerSecond:N0} /s (-{delta:F1}%)."; + } + } + + private static double? CalculateDurationRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + return current / baseline.Value; + } + + private static double? CalculateThroughputRatio(double current, double? baseline) + { + if (!baseline.HasValue || baseline.Value <= 0d) + { + return null; + } + + if (current <= 0d) + { + return double.PositiveInfinity; + } + + return baseline.Value / current; + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/PrometheusWriter.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/PrometheusWriter.cs new file mode 100644 index 00000000..ea6f445a --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/Reporting/PrometheusWriter.cs @@ -0,0 +1,83 @@ +using System.Globalization; +using System.Text; + +namespace StellaOps.Bench.PolicyEngine.Reporting; + +internal static class PrometheusWriter +{ + public static void Write(string path, IReadOnlyList reports) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var builder = new StringBuilder(); + builder.AppendLine("# HELP policy_engine_bench_duration_ms Policy Engine benchmark duration metrics (milliseconds)."); + builder.AppendLine("# TYPE policy_engine_bench_duration_ms gauge"); + builder.AppendLine("# HELP policy_engine_bench_throughput_per_sec Policy Engine benchmark throughput metrics (findings per second)."); + builder.AppendLine("# TYPE policy_engine_bench_throughput_per_sec gauge"); + builder.AppendLine("# HELP policy_engine_bench_allocation_mb Policy Engine benchmark allocation metrics (megabytes)."); + builder.AppendLine("# TYPE policy_engine_bench_allocation_mb gauge"); + + foreach (var report in reports) + { + var scenarioLabel = Escape(report.Result.Id); + AppendMetric(builder, "policy_engine_bench_mean_ms", scenarioLabel, report.Result.MeanMs); + AppendMetric(builder, "policy_engine_bench_p95_ms", scenarioLabel, report.Result.P95Ms); + AppendMetric(builder, "policy_engine_bench_max_ms", scenarioLabel, report.Result.MaxMs); + AppendMetric(builder, "policy_engine_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs); + + AppendMetric(builder, "policy_engine_bench_mean_throughput_per_sec", scenarioLabel, report.Result.MeanThroughputPerSecond); + AppendMetric(builder, "policy_engine_bench_min_throughput_per_sec", scenarioLabel, report.Result.MinThroughputPerSecond); + AppendMetric(builder, "policy_engine_bench_min_throughput_threshold_per_sec", scenarioLabel, report.Result.MinThroughputThresholdPerSecond); + + AppendMetric(builder, "policy_engine_bench_max_allocated_mb", scenarioLabel, report.Result.MaxAllocatedMb); + AppendMetric(builder, "policy_engine_bench_max_allocated_threshold_mb", scenarioLabel, report.Result.MaxAllocatedThresholdMb); + + if (report.Baseline is { } baseline) + { + AppendMetric(builder, "policy_engine_bench_baseline_max_ms", scenarioLabel, baseline.MaxMs); + AppendMetric(builder, "policy_engine_bench_baseline_mean_ms", scenarioLabel, baseline.MeanMs); + AppendMetric(builder, "policy_engine_bench_baseline_min_throughput_per_sec", scenarioLabel, baseline.MinThroughputPerSecond); + } + + if (report.DurationRegressionRatio is { } durationRatio) + { + AppendMetric(builder, "policy_engine_bench_duration_regression_ratio", scenarioLabel, durationRatio); + } + + if (report.ThroughputRegressionRatio is { } throughputRatio) + { + AppendMetric(builder, "policy_engine_bench_throughput_regression_ratio", scenarioLabel, throughputRatio); + } + + AppendMetric(builder, "policy_engine_bench_regression_limit", scenarioLabel, report.RegressionLimit); + AppendMetric(builder, "policy_engine_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0); + } + + File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8); + } + + private static void AppendMetric(StringBuilder builder, string metric, string scenario, double? value) + { + if (!value.HasValue) + { + return; + } + + builder.Append(metric); + builder.Append("{scenario=\""); + builder.Append(scenario); + builder.Append("\"} "); + builder.AppendLine(value.Value.ToString("G17", CultureInfo.InvariantCulture)); + } + + private static string Escape(string value) => + value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/ScenarioResult.cs b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/ScenarioResult.cs new file mode 100644 index 00000000..8486133c --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/ScenarioResult.cs @@ -0,0 +1,110 @@ +using System.Globalization; + +namespace StellaOps.Bench.PolicyEngine; + +internal sealed record ScenarioResult( + string Id, + string Label, + int Iterations, + int FindingCount, + double MeanMs, + double P95Ms, + double MaxMs, + double MeanThroughputPerSecond, + double MinThroughputPerSecond, + double MaxAllocatedMb, + double? ThresholdMs, + double? MinThroughputThresholdPerSecond, + double? MaxAllocatedThresholdMb) +{ + public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28]; + public string FindingsColumn => FindingCount.ToString("N0", CultureInfo.InvariantCulture).PadLeft(12); + public string MeanColumn => MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + public string P95Column => P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + public string MaxColumn => MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); + public string MinThroughputColumn => (MinThroughputPerSecond / 1_000d).ToString("F2", CultureInfo.InvariantCulture).PadLeft(11); + public string AllocatedColumn => MaxAllocatedMb.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9); +} + +internal readonly record struct DurationStatistics(double MeanMs, double P95Ms, double MaxMs) +{ + public static DurationStatistics From(IReadOnlyList durations) + { + if (durations.Count == 0) + { + return new DurationStatistics(0, 0, 0); + } + + var sorted = durations.ToArray(); + Array.Sort(sorted); + + var total = 0d; + foreach (var value in durations) + { + total += value; + } + + var mean = total / durations.Count; + var p95 = Percentile(sorted, 95); + var max = sorted[^1]; + + return new DurationStatistics(mean, p95, max); + } + + private static double Percentile(IReadOnlyList sorted, double percentile) + { + if (sorted.Count == 0) + { + return 0; + } + + var rank = (percentile / 100d) * (sorted.Count - 1); + var lower = (int)Math.Floor(rank); + var upper = (int)Math.Ceiling(rank); + var weight = rank - lower; + + if (upper >= sorted.Count) + { + return sorted[lower]; + } + + return sorted[lower] + weight * (sorted[upper] - sorted[lower]); + } +} + +internal readonly record struct ThroughputStatistics(double MeanPerSecond, double MinPerSecond) +{ + public static ThroughputStatistics From(IReadOnlyList values) + { + if (values.Count == 0) + { + return new ThroughputStatistics(0, 0); + } + + var total = 0d; + var min = double.MaxValue; + + foreach (var value in values) + { + total += value; + min = Math.Min(min, value); + } + + var mean = total / values.Count; + return new ThroughputStatistics(mean, min); + } +} + +internal readonly record struct AllocationStatistics(double MaxAllocatedMb) +{ + public static AllocationStatistics From(IReadOnlyList values) + { + var max = 0d; + foreach (var value in values) + { + max = Math.Max(max, value); + } + + return new AllocationStatistics(max); + } +} diff --git a/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/StellaOps.Bench.PolicyEngine.csproj b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/StellaOps.Bench.PolicyEngine.csproj new file mode 100644 index 00000000..dbe05703 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/StellaOps.Bench.PolicyEngine.csproj @@ -0,0 +1,14 @@ + + + Exe + net10.0 + enable + enable + preview + true + + + + + + diff --git a/src/StellaOps.Bench/PolicyEngine/baseline.csv b/src/StellaOps.Bench/PolicyEngine/baseline.csv new file mode 100644 index 00000000..79cdb0d4 --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/baseline.csv @@ -0,0 +1,2 @@ +scenario,iterations,findings,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb +policy_eval_baseline,3,1000000,1109.3542,1257.7493,1280.1721,912094.5581,781144.9726,563.6901 diff --git a/src/StellaOps.Bench/PolicyEngine/config.json b/src/StellaOps.Bench/PolicyEngine/config.json new file mode 100644 index 00000000..e5242fdb --- /dev/null +++ b/src/StellaOps.Bench/PolicyEngine/config.json @@ -0,0 +1,19 @@ +{ + "iterations": 3, + "thresholdMs": 20000, + "minThroughputPerSecond": 60000, + "maxAllocatedMb": 900, + "scenarios": [ + { + "id": "policy_eval_baseline", + "label": "Policy evaluation (100k components, 1M findings)", + "policyPath": "docs/examples/policies/baseline.yaml", + "componentCount": 100000, + "advisoriesPerComponent": 10, + "seed": 20251026, + "thresholdMs": 20000, + "minThroughputPerSecond": 60000, + "maxAllocatedMb": 900 + } + ] +} diff --git a/src/StellaOps.Bench/TASKS.md b/src/StellaOps.Bench/TASKS.md index 2f99914b..7db23741 100644 --- a/src/StellaOps.Bench/TASKS.md +++ b/src/StellaOps.Bench/TASKS.md @@ -5,35 +5,36 @@ | BENCH-SCANNER-10-001 | DONE | Bench Guild, Scanner Team | SCANNER-ANALYZERS-LANG-10-303 | Analyzer microbench harness (node_modules, site-packages) + baseline CSV. | Harness committed under `src/StellaOps.Bench/Scanner.Analyzers`; baseline CSV recorded; CI job publishes results. | | BENCH-SCANNER-10-002 | DONE (2025-10-21) | Bench Guild, Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301..309 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | Harness executes analyzer assemblies end-to-end; updated baseline committed; CI trend doc linked. | | BENCH-IMPACT-16-001 | TODO | Bench Guild, Scheduler Team | SCHED-IMPACT-16-301 | ImpactIndex throughput bench (resolve 10k productKeys) + RAM profile. | Benchmark script ready; baseline metrics recorded; alert thresholds defined. | -| BENCH-NOTIFY-15-001 | TODO | Bench Guild, Notify Team | NOTIFY-ENGINE-15-301 | Notify dispatch throughput bench (vary rule density) with results CSV. | Bench executed; results stored; regression alert configured. | +| BENCH-NOTIFY-15-001 | DONE (2025-10-26) | Bench Guild, Notify Team | NOTIFY-ENGINE-15-301 | Notify dispatch throughput bench (vary rule density) with results CSV. | Bench executed; results stored; regression alert configured. | ## Policy Engine v2 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| BENCH-POLICY-20-001 | TODO | Bench Guild, Policy Guild | POLICY-ENGINE-20-002, POLICY-ENGINE-20-006 | Build policy evaluation benchmark suite (100k components, 1M advisories) capturing latency, throughput, memory. | Bench harness committed; baseline metrics recorded; ties into CI dashboards. | -| BENCH-POLICY-20-002 | TODO | Bench Guild, Policy Guild, Scheduler Guild | BENCH-POLICY-20-001, SCHED-WORKER-20-302 | Add incremental run benchmark measuring delta evaluation vs full; capture SLA compliance. | Incremental bench executed; results stored; regression alerts configured. | +| BENCH-POLICY-20-001 | DONE (2025-10-26) | Bench Guild, Policy Guild | POLICY-ENGINE-20-002, POLICY-ENGINE-20-006 | Build policy evaluation benchmark suite (100k components, 1M advisories) capturing latency, throughput, memory. | Bench harness committed; baseline metrics recorded; ties into CI dashboards. | +> 2025-10-26: Added `StellaOps.Bench.PolicyEngine` harness, synthetic dataset generator, baseline + Prom/JSON outputs; default thresholds cover latency/throughput/allocation. +| BENCH-POLICY-20-002 | BLOCKED (waiting on SCHED-WORKER-20-302) | Bench Guild, Policy Guild, Scheduler Guild | BENCH-POLICY-20-001, SCHED-WORKER-20-302 | Add incremental run benchmark measuring delta evaluation vs full; capture SLA compliance. | Incremental bench executed; results stored; regression alerts configured. | +> 2025-10-26: Scheduler delta targeting (SCHED-WORKER-20-302) not implemented; incremental bench paused until worker emits delta input stream. ## Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| BENCH-GRAPH-21-001 | TODO | Bench Guild, Cartographer Guild | CARTO-GRAPH-21-004, CARTO-GRAPH-21-006 | Build graph viewport/path benchmark harness simulating 50k/100k nodes; record latency, memory, tile cache hit rates. | Harness committed; baseline metrics logged; integrates with perf dashboards. | -| BENCH-GRAPH-21-002 | TODO | Bench Guild, UI Guild | BENCH-GRAPH-21-001, UI-GRAPH-21-001 | Add headless UI load benchmark (Playwright) for graph canvas interactions to track render times and FPS budgets. | Benchmark runs in CI; results exported; alert thresholds defined. | +| BENCH-GRAPH-21-001 | DOING (2025-10-27) | Bench Guild, Graph Platform Guild | GRAPH-API-28-003, GRAPH-INDEX-28-006 | Build graph viewport/path benchmark harness (50k/100k nodes) measuring Graph API/Indexer latency, memory, and tile cache hit rates. *(Executed within Sprint 28 Graph program).* | Harness committed; baseline metrics logged; integrates with perf dashboards. | +| BENCH-GRAPH-21-002 | TODO | Bench Guild, UI Guild | BENCH-GRAPH-21-001, UI-GRAPH-24-001 | Add headless UI load benchmark (Playwright) for graph canvas interactions to track render times and FPS budgets. *(Executed within Sprint 28 Graph program).* | Benchmark runs in CI; results exported; alert thresholds defined. | ## Link-Not-Merge v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| BENCH-LNM-22-001 | TODO | Bench Guild, Concelier Guild | CONCELIER-LNM-21-002 | Create ingest benchmark simulating 500 advisory observations/sec, measuring correlator latency and Mongo throughput; publish baseline metrics. | Harness added; baseline stored; alerts wired for SLA breach. | -| BENCH-LNM-22-002 | TODO | Bench Guild, Excititor Guild | EXCITITOR-LNM-21-002 | Build VEX ingestion/correlation perf test focusing on alias/product matching and event emission latency. | Benchmark executed; metrics captured; CI integration established. | +| BENCH-LNM-22-001 | DONE (2025-10-26) | Bench Guild, Concelier Guild | CONCELIER-LNM-21-002 | Create ingest benchmark simulating 500 advisory observations/sec, measuring correlator latency and Mongo throughput; publish baseline metrics. | Harness added; baseline stored; alerts wired for SLA breach. | +| BENCH-LNM-22-002 | DONE (2025-10-26) | Bench Guild, Excititor Guild | EXCITITOR-LNM-21-002 | Build VEX ingestion/correlation perf test focusing on alias/product matching and event emission latency. | Benchmark executed; metrics captured; CI integration established. | ## Graph & Vuln Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| BENCH-GRAPH-24-001 | TODO | Bench Guild, SBOM Service Guild | SBOM-GRAPH-24-002 | Develop SBOM graph performance benchmark measuring build time, memory, and cache warm latency for 40k-node assets. | Benchmark runs in CI; baseline metrics recorded; alerts configured. | -| BENCH-GRAPH-24-002 | TODO | Bench Guild, UI Guild | UI-GRAPH-24-001..005 | Implement UI interaction benchmarks (filter/zoom/table operations) citing p95 latency; integrate with perf dashboards. | UI perf metrics collected; thresholds enforced; documentation updated. | +| BENCH-GRAPH-24-002 | TODO | Bench Guild, UI Guild | UI-GRAPH-24-001, UI-GRAPH-24-002 | Implement UI interaction benchmarks (filter/zoom/table operations) citing p95 latency; integrate with perf dashboards. | UI perf metrics collected; thresholds enforced; documentation updated. | ## Reachability v1 diff --git a/src/StellaOps.Cartographer/TASKS.md b/src/StellaOps.Cartographer/TASKS.md index dd6ed4aa..7130fa99 100644 --- a/src/StellaOps.Cartographer/TASKS.md +++ b/src/StellaOps.Cartographer/TASKS.md @@ -1,12 +1,6 @@ # Cartographer Task Board — Epic 3: Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| CARTO-GRAPH-21-001 | TODO | Cartographer Guild | SBOM-SERVICE-21-001 | Define graph storage schema: snapshot catalog, node/edge collections, overlay tables, sharding strategy, indexes, and retention rules. Document migration/rollback. | Mongo collections + indexes created via bootstrapper; schema doc drafted; integration tests verify tenant isolation and sharding. | -| CARTO-GRAPH-21-002 | TODO | Cartographer Guild | CARTO-GRAPH-21-001 | Implement SBOM projection reader consuming normalized CycloneDX/SPDX from SBOM Service; validate fixtures and handle entrypoint tagging. | Reader ingests sample SBOMs (CycloneDX/SPDX); entrypoints resolved; unit tests cover malformed SBOM handling. | -| CARTO-GRAPH-21-003 | TODO | Cartographer Guild | CARTO-GRAPH-21-002 | Build graph constructor deduping PURLs, emitting typed nodes/edges with metadata (licenses, supplier, hashes, scope). | Graph snapshots generated deterministically for fixtures; duplicate suppression verified; stats recorded. | -| CARTO-GRAPH-21-004 | TODO | Cartographer Guild | CARTO-GRAPH-21-003 | Implement layout & tiling pipeline (global force layout + zoom tiles) and persist tiles to blob/object storage with manifests. | Layout tiles produced for fixtures; bbox metadata recorded; perf target (<10 min for 50k nodes) documented. | -| CARTO-GRAPH-21-005 | TODO | Cartographer Guild | CARTO-GRAPH-21-003, POLICY-ENGINE-30-001 | Overlay worker hydrating policy effective findings/VEX data, computing path relevance from entrypoints, and supporting simulation overlays. | Overlay documents created for selected policies; path relevance validated via unit tests; SLA metric recorded. | -| CARTO-GRAPH-21-006 | TODO | Cartographer Guild, BE-Base Platform Guild | CARTO-GRAPH-21-005 | Expose Cartographer APIs (versions, viewport tiles, node lookup, path queries, filters, diff, export, simulate) with pagination and streaming support. | API endpoints documented, tested, and OpenAPI published; large responses stream without timeouts; error codes mapped. | -| CARTO-GRAPH-21-007 | TODO | Cartographer Guild, Scheduler Guild | CARTO-GRAPH-21-005, SCHED-WORKER-21-201 | Build backfill + incremental overlay jobs listening to policy/SBOM change events; ensure eventual consistency SLA (<2 min). | Change stream consumers enqueue work; retries/backoff implemented; observability metrics emit overlay lag. | -| CARTO-GRAPH-21-008 | TODO | Cartographer Guild, QA Guild | CARTO-GRAPH-21-004..007 | Add unit/property/integration tests, synthetic 50k/100k perf suites, chaos scenarios (missing overlays, cyclic graphs), and determinism checks. | Test suite green; perf benchmarks logged; regression pipeline enforces determinism. | -| CARTO-GRAPH-21-009 | TODO | Cartographer Guild, DevOps Guild | CARTO-GRAPH-21-006 | Provide deployment artefacts (Helm/Compose), configuration docs, and Offline Kit bundle notes for Cartographer. | Deployment descriptors merged; docs updated; offline kit includes seeds/layout caches; smoke tests added. | +| CARTO-GRAPH-21-010 | TODO | Cartographer Guild | AUTH-GRAPH-21-001 | Replace hard-coded `graph:*` scope strings in Cartographer services/clients with `StellaOpsScopes` constants; document new dependency. | All scope checks reference `StellaOpsScopes`; documentation updated; unit tests adjusted if needed. | + +> 2025-10-26 — Note: awaiting Cartographer service bootstrap. Keep this task open until Cartographer routes exist so we can swap to `StellaOpsScopes` immediately. diff --git a/src/StellaOps.Cli.Tests/Plugins/CliCommandModuleLoaderTests.cs b/src/StellaOps.Cli.Tests/Plugins/CliCommandModuleLoaderTests.cs index 9702c073..f2c4facc 100644 --- a/src/StellaOps.Cli.Tests/Plugins/CliCommandModuleLoaderTests.cs +++ b/src/StellaOps.Cli.Tests/Plugins/CliCommandModuleLoaderTests.cs @@ -3,6 +3,7 @@ using System.CommandLine; using System.IO; using System.Threading; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using StellaOps.Cli.Configuration; using StellaOps.Cli.Plugins; diff --git a/src/StellaOps.Cli/StellaOps.Cli.csproj b/src/StellaOps.Cli/StellaOps.Cli.csproj index d047e632..94b3629c 100644 --- a/src/StellaOps.Cli/StellaOps.Cli.csproj +++ b/src/StellaOps.Cli/StellaOps.Cli.csproj @@ -1,41 +1,41 @@ - - - - Exe - net10.0 - enable - enable - - + + + + Exe + net10.0 + enable + enable + + - - - - - - - - + + + + + + + + - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/StellaOps.Cli/TASKS.md b/src/StellaOps.Cli/TASKS.md index 8283a002..b53b9efb 100644 --- a/src/StellaOps.Cli/TASKS.md +++ b/src/StellaOps.Cli/TASKS.md @@ -2,8 +2,11 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | CLI-AOC-19-001 | TODO | DevEx/CLI Guild | CONCELIER-WEB-AOC-19-001, EXCITITOR-WEB-AOC-19-001 | Implement `stella sources ingest --dry-run` printing would-write payloads with forbidden field scan results and guard status. | Command displays diff-safe JSON, highlights forbidden fields, exits non-zero on guard violation, and has unit tests. | +> Docs ready (2025-10-26): Reference behaviour/spec in `docs/cli/cli-reference.md` §2 and AOC reference §5. | CLI-AOC-19-002 | TODO | DevEx/CLI Guild | CLI-AOC-19-001 | Add `stella aoc verify` command supporting `--since`/`--limit`, mapping `ERR_AOC_00x` to exit codes, with JSON/table output. | Command integrates with both services, exit codes documented, regression tests green. | +> Docs ready (2025-10-26): CLI guide §3 covers options/exit codes; deployment doc `docs/deploy/containers.md` describes required verifier user. | CLI-AOC-19-003 | TODO | Docs/CLI Guild | CLI-AOC-19-001, CLI-AOC-19-002 | Update CLI reference and quickstart docs to cover new commands, exit codes, and offline verification workflows. | Docs updated; examples recorded; release notes mention new commands. | +> Docs note (2025-10-26): `docs/cli/cli-reference.md` now describes both commands, exit codes, and offline usage—sync help text once implementation lands. ## Policy Engine v2 @@ -11,15 +14,13 @@ |----|--------|----------|------------|-------------|---------------| | CLI-POLICY-20-001 | TODO | DevEx/CLI Guild | WEB-POLICY-20-001 | Add `stella policy new|edit|submit|approve` commands with local editor integration, version pinning, and approval workflow wiring. | Commands round-trip policy drafts with temp files; approval requires correct scopes; unit tests cover happy/error paths. | | CLI-POLICY-20-002 | TODO | DevEx/CLI Guild | CLI-POLICY-20-001, WEB-POLICY-20-001, WEB-POLICY-20-002 | Implement `stella policy simulate` with SBOM/env arguments and diff output (table/JSON), handling exit codes for `ERR_POL_*`. | Simulation outputs deterministic diffs; JSON schema documented; tests validate exit codes + piping of env variables. | +> 2025-10-26: Scheduler Models expose canonical run/diff schemas (`src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md`). Schema exporter lives at `scripts/export-policy-schemas.sh`; wire schema validation once DevOps publishes artifacts (see DEVOPS-POLICY-20-004). | CLI-POLICY-20-003 | TODO | DevEx/CLI Guild, Docs Guild | CLI-POLICY-20-002, WEB-POLICY-20-003, DOCS-POLICY-20-006 | Extend `stella findings ls|get` commands for policy-filtered retrieval with pagination, severity filters, and explain output. | Commands stream paginated results; explain view renders rationale entries; docs/help updated; end-to-end tests cover filters. | ## Graph Explorer v1 | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| CLI-GRAPH-21-001 | TODO | DevEx/CLI Guild, Cartographer Guild | CARTO-GRAPH-21-006, WEB-GRAPH-21-001 | Implement `stella sbom graph build/export/query/diff` commands with RBAC-aware behavior, streaming outputs, and offline mode. | Commands execute against local fixtures; GraphML/JSONL exports match API; scope errors mapped to `ERR_Graph_*`; unit tests added. | -| CLI-GRAPH-21-002 | TODO | DevEx/CLI Guild | CLI-GRAPH-21-001, WEB-GRAPH-21-004 | Add path query and simulation options (k-shortest paths, policy selection, filters) with JSON output suitable for CI. | Path query returns expected JSON schema; simulation overlay toggles; regression tests cover CLI-to-API contract. | -| CLI-GRAPH-21-003 | TODO | DevEx/CLI Guild, Docs Guild | CLI-GRAPH-21-001, DOCS-GRAPH-21-005 | Document CLI usage and provide golden fixtures for CI; ensure exit codes align with `ERR_Graph_*`. | Docs updated; fixtures stored under `samples/graph/`; CI job runs CLI smoke; exit codes verified. | ## Link-Not-Merge v1 @@ -40,8 +41,6 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| CLI-GRAPH-24-001 | TODO | DevEx/CLI Guild | WEB-GRAPH-24-001 | Add `stella graph show|search|diff` commands with JSON/table outputs, pagination, depth controls, and RBAC-aware error handling. | Commands return graph data; diff output validated with fixtures; unit tests cover pagination/error cases. | -| CLI-GRAPH-24-002 | TODO | DevEx/CLI Guild | WEB-GRAPH-24-002 | Implement `stella graph simulate upgrade|policy` commands including progress feedback, diff summary, and exit codes for failure/timeouts. | Simulation commands integrate with API; regression tests exercise upgrade/policy scenarios; docs updated. | ## Exceptions v1 @@ -151,6 +150,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | CLI-AIRGAP-56-001 | TODO | DevEx/CLI Guild | MIRROR-CRT-56-001, AIRGAP-IMP-56-001 | Implement `stella mirror create|verify` and `stella airgap verify` commands with DSSE/TUF results, dry-run mode, and deterministic manifests. | Commands produce deterministic bundles; verify outputs structured DSSE/TUF results; integration tests cover tampering scenarios. | +| CLI-AIRGAP-56-002 | TODO | DevEx/CLI Guild | CLI-OBS-50-001, AIRGAP-IMP-56-001 | Ensure telemetry propagation under sealed mode (no remote exporters) while preserving correlation IDs; add label `AirGapped-Phase-1`. | CLI traces flow via local exporters in sealed mode; correlation IDs still printed; tests cover sealed toggle + fallback. | | CLI-AIRGAP-57-001 | TODO | DevEx/CLI Guild | CLI-AIRGAP-56-001, AIRGAP-IMP-58-001 | Add `stella airgap import` with diff preview, bundle scope selection (`--tenant`, `--global`), audit logging, and progress reporting. | Import updates catalog; diff preview rendered; audit entries include bundle ID + scope; tests cover idempotent re-import. | | CLI-AIRGAP-57-002 | TODO | DevEx/CLI Guild | CLI-AIRGAP-56-001, AIRGAP-CTL-56-002 | Provide `stella airgap seal|status` commands surfacing sealing state, drift, staleness metrics, and remediation guidance with safe confirmation prompts. | Status command prints drift/staleness; seal requires confirmation + scope; integration tests cover RBAC denials. | | CLI-AIRGAP-58-001 | TODO | DevEx/CLI Guild, Evidence Locker Guild | CLI-AIRGAP-57-001, CLI-FORENSICS-54-001 | Implement `stella airgap export evidence` helper for portable evidence packages, including checksum manifest and verification. | Command generates portable bundle; verification step validates signatures; docs/help updated with examples. | diff --git a/src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj b/src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj index 0096d321..93b18a9f 100644 --- a/src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj +++ b/src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj @@ -1,20 +1,20 @@ - - - net10.0 - enable - enable - - - - + + + net10.0 + enable + enable + + + + - - - - - - - - - + + + + + + + + + diff --git a/src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj b/src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj index a85281ba..1ba04e65 100644 --- a/src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj +++ b/src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - + + + net10.0 + preview + enable + enable + true + - - - - - - - - - - + + + + + + + + + + diff --git a/src/StellaOps.Concelier.Core/TASKS.md b/src/StellaOps.Concelier.Core/TASKS.md index 43699a75..00604e35 100644 --- a/src/StellaOps.Concelier.Core/TASKS.md +++ b/src/StellaOps.Concelier.Core/TASKS.md @@ -3,9 +3,14 @@ | ID | Status | Owner(s) | Depends on | Notes | |---|---|---|---|---| | CONCELIER-CORE-AOC-19-001 `AOC write guard` | TODO | Concelier Core Guild | WEB-AOC-19-001 | Implement repository interceptor that inspects write payloads for forbidden AOC keys, validates provenance/signature presence, and maps violations to `ERR_AOC_00x`. | +> Docs alignment (2025-10-26): Behaviour/spec captured in `docs/ingestion/aggregation-only-contract.md` and architecture overview §2. | CONCELIER-CORE-AOC-19-002 `Deterministic linkset extraction` | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-001 | Build canonical linkset mappers for CVE/GHSA/PURL/CPE/reference extraction from upstream raw payloads, ensuring reconciled-from metadata is tracked and deterministic. | +> Docs alignment (2025-10-26): Linkset expectations detailed in AOC reference §4 and policy-engine architecture §2.1. | CONCELIER-CORE-AOC-19-003 `Idempotent append-only upsert` | TODO | Concelier Core Guild | CONCELIER-STORE-AOC-19-002 | Implement idempotent upsert path using `(vendor, upstreamId, contentHash, tenant)` key, emitting supersedes pointers for new revisions and preventing duplicate inserts. | +> Docs alignment (2025-10-26): Deployment guide + observability guide describe supersedes metrics; ensure implementation emits `aoc_violation_total` on failure. | CONCELIER-CORE-AOC-19-004 `Remove ingestion normalization` | TODO | Concelier Core Guild | CONCELIER-CORE-AOC-19-002, POLICY-AOC-19-003 | Strip normalization/dedup/severity logic from ingestion pipelines, delegate derived computations to Policy Engine, and update exporters/tests to consume raw documents only. | +> Docs alignment (2025-10-26): Architecture overview emphasises policy-only derivation; coordinate with Policy Engine guild for rollout. +| CONCELIER-CORE-AOC-19-013 `Authority tenant scope smoke coverage` | TODO | Concelier Core Guild | AUTH-AOC-19-002 | Extend Concelier smoke/e2e fixtures to configure `requiredTenants` and assert cross-tenant rejection with updated Authority tokens. | Coordinate deliverable so Authority docs (`AUTH-AOC-19-003`) can close once tests are in place. | ## Policy Engine v2 @@ -41,7 +46,7 @@ | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| CONCELIER-GRAPH-24-001 `Advisory overlay inputs` | TODO | Concelier Core Guild | CONCELIER-POLICY-23-001 | Expose raw advisory observations/linksets with tenant filters for overlay services; no derived counts/severity in ingestion. | +| CONCELIER-GRAPH-24-001 `Advisory overlay inputs` | DOING (2025-10-27) | Concelier Core Guild | CONCELIER-POLICY-23-001 | Expose raw advisory observations/linksets with tenant filters for overlay services; no derived counts/severity in ingestion. | ## Reachability v1 diff --git a/src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj b/src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj index 967a1a9e..f8210e66 100644 --- a/src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj +++ b/src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj @@ -1,24 +1,24 @@ - - - - net10.0 - preview - enable - enable - true - - - - + + + + net10.0 + preview + enable + enable + true + + + + - - - - + + + + diff --git a/src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj b/src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj index eb183fc4..a12b1a84 100644 --- a/src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj @@ -1,22 +1,22 @@ - - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - - + + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs b/src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs index 6a48a98d..98f23499 100644 --- a/src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs +++ b/src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs @@ -238,25 +238,8 @@ public sealed class AdvisoryMergeService } } - var payload = new ConflictDetailPayload( - detail.ConflictType, - detail.Reason, - detail.PrimarySources, - detail.PrimaryRank, - detail.SuppressedSources, - detail.SuppressedRank, - detail.PrimaryValue, - detail.SuppressedValue); - - var explainer = new MergeConflictExplainerPayload( - payload.Type, - payload.Reason, - payload.PrimarySources, - payload.PrimaryRank, - payload.SuppressedSources, - payload.SuppressedRank, - payload.PrimaryValue, - payload.SuppressedValue); + var payload = ConflictDetailPayload.FromDetail(detail); + var explainer = payload.ToExplainer(); var canonicalJson = explainer.ToCanonicalJson(); var document = JsonDocument.Parse(canonicalJson); diff --git a/src/StellaOps.Concelier.Merge/Services/ConflictDetailPayload.cs b/src/StellaOps.Concelier.Merge/Services/ConflictDetailPayload.cs new file mode 100644 index 00000000..d00e4b74 --- /dev/null +++ b/src/StellaOps.Concelier.Merge/Services/ConflictDetailPayload.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Merge.Services; + +/// +/// Canonical conflict detail used to materialize structured payloads for persistence and explainers. +/// +public sealed record ConflictDetailPayload( + string Type, + string Reason, + IReadOnlyList PrimarySources, + int PrimaryRank, + IReadOnlyList SuppressedSources, + int SuppressedRank, + string? PrimaryValue, + string? SuppressedValue) +{ + public static ConflictDetailPayload FromDetail(MergeConflictDetail detail) + { + ArgumentNullException.ThrowIfNull(detail); + + return new ConflictDetailPayload( + detail.ConflictType, + detail.Reason, + detail.PrimarySources, + detail.PrimaryRank, + detail.SuppressedSources, + detail.SuppressedRank, + detail.PrimaryValue, + detail.SuppressedValue); + } + + public MergeConflictExplainerPayload ToExplainer() => + new( + Type, + Reason, + PrimarySources, + PrimaryRank, + SuppressedSources, + SuppressedRank, + PrimaryValue, + SuppressedValue); +} diff --git a/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj b/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj index 94468bc2..4e9ff539 100644 --- a/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj +++ b/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj @@ -1,14 +1,14 @@ - - - - - net10.0 - enable - enable - - + + + + + net10.0 + enable + enable + + - + diff --git a/src/StellaOps.Concelier.Models.Tests/Observations/AdvisoryObservationTests.cs b/src/StellaOps.Concelier.Models.Tests/Observations/AdvisoryObservationTests.cs new file mode 100644 index 00000000..1366d42d --- /dev/null +++ b/src/StellaOps.Concelier.Models.Tests/Observations/AdvisoryObservationTests.cs @@ -0,0 +1,61 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json.Nodes; +using StellaOps.Concelier.Models.Observations; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests.Observations; + +public sealed class AdvisoryObservationTests +{ + [Fact] + public void Constructor_NormalizesTenantAndCollections() + { + var source = new AdvisoryObservationSource("Vendor", "Stream", "https://example.com/api", "1.2.3"); + var signature = new AdvisoryObservationSignature(true, "pgp", "key1", "sig"); + var upstream = new AdvisoryObservationUpstream( + upstreamId: "CVE-2025-1234", + documentVersion: "2025-10-01", + fetchedAt: DateTimeOffset.Parse("2025-10-01T01:00:00Z"), + receivedAt: DateTimeOffset.Parse("2025-10-01T01:00:05Z"), + contentHash: "sha256:abc", + signature: signature); + + var content = new AdvisoryObservationContent("CSAF", "2.0", JsonNode.Parse("{\"foo\":1}")!); + + var linkset = new AdvisoryObservationLinkset( + aliases: new[] { " Cve-2025-1234 ", "cve-2025-1234" }, + purls: new[] { "pkg:generic/foo@1.0.0", "pkg:generic/foo@1.0.0" }, + cpes: new[] { "cpe:/a:vendor:product:1" }, + references: new[] + { + new AdvisoryObservationReference("ADVISORY", "https://example.com/advisory"), + new AdvisoryObservationReference("advisory", "https://example.com/advisory") + }); + + var attributes = ImmutableDictionary.CreateRange(new Dictionary + { + [" region "] = "emea", + ["pipeline"] = "daily" + }); + + var observation = new AdvisoryObservation( + observationId: " tenant-a:CVE-2025-1234:1 ", + tenant: " Tenant-A ", + source: source, + upstream: upstream, + content: content, + linkset: linkset, + createdAt: DateTimeOffset.Parse("2025-10-01T01:00:06Z"), + attributes: attributes); + + Assert.Equal("tenant-a:CVE-2025-1234:1", observation.ObservationId); + Assert.Equal("tenant-a", observation.Tenant); + Assert.Equal("Vendor", observation.Source.Vendor); + Assert.Equal(new[] { "cpe:/a:vendor:product:1" }, observation.Linkset.Cpes); + Assert.Single(observation.Linkset.References); + Assert.Equal("https://example.com/advisory", observation.Linkset.References[0].Url); + Assert.Equal(DateTimeOffset.Parse("2025-10-01T01:00:06Z"), observation.CreatedAt); + Assert.Equal("emea", observation.Attributes["region"]); + } +} diff --git a/src/StellaOps.Concelier.Models/Advisory.cs b/src/StellaOps.Concelier.Models/Advisory.cs index d522d329..dce9629f 100644 --- a/src/StellaOps.Concelier.Models/Advisory.cs +++ b/src/StellaOps.Concelier.Models/Advisory.cs @@ -1,8 +1,8 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json.Serialization; - -namespace StellaOps.Concelier.Models; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Models; /// /// Canonical advisory document produced after merge. Collections are pre-sorted for deterministic serialization. diff --git a/src/StellaOps.Concelier.Models/Observations/AdvisoryObservation.cs b/src/StellaOps.Concelier.Models/Observations/AdvisoryObservation.cs new file mode 100644 index 00000000..a855004f --- /dev/null +++ b/src/StellaOps.Concelier.Models/Observations/AdvisoryObservation.cs @@ -0,0 +1,283 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace StellaOps.Concelier.Models.Observations; + +public sealed record AdvisoryObservation +{ + public AdvisoryObservation( + string observationId, + string tenant, + AdvisoryObservationSource source, + AdvisoryObservationUpstream upstream, + AdvisoryObservationContent content, + AdvisoryObservationLinkset linkset, + DateTimeOffset createdAt, + ImmutableDictionary? attributes = null) + { + ObservationId = Validation.EnsureNotNullOrWhiteSpace(observationId, nameof(observationId)); + Tenant = Validation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)).ToLowerInvariant(); + Source = source ?? throw new ArgumentNullException(nameof(source)); + Upstream = upstream ?? throw new ArgumentNullException(nameof(upstream)); + Content = content ?? throw new ArgumentNullException(nameof(content)); + Linkset = linkset ?? throw new ArgumentNullException(nameof(linkset)); + CreatedAt = createdAt.ToUniversalTime(); + Attributes = NormalizeAttributes(attributes); + } + + public string ObservationId { get; } + + public string Tenant { get; } + + public AdvisoryObservationSource Source { get; } + + public AdvisoryObservationUpstream Upstream { get; } + + public AdvisoryObservationContent Content { get; } + + public AdvisoryObservationLinkset Linkset { get; } + + public DateTimeOffset CreatedAt { get; } + + public ImmutableDictionary Attributes { get; } + + private static ImmutableDictionary NormalizeAttributes(ImmutableDictionary? attributes) + { + if (attributes is null || attributes.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var pair in attributes) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + builder[pair.Key.Trim()] = pair.Value; + } + + return builder.ToImmutable(); + } +} + +public sealed record AdvisoryObservationSource +{ + public AdvisoryObservationSource( + string vendor, + string stream, + string api, + string? collectorVersion = null) + { + Vendor = Validation.EnsureNotNullOrWhiteSpace(vendor, nameof(vendor)); + Stream = Validation.EnsureNotNullOrWhiteSpace(stream, nameof(stream)); + Api = Validation.EnsureNotNullOrWhiteSpace(api, nameof(api)); + CollectorVersion = Validation.TrimToNull(collectorVersion); + } + + public string Vendor { get; } + + public string Stream { get; } + + public string Api { get; } + + public string? CollectorVersion { get; } +} + +public sealed record AdvisoryObservationSignature +{ + public AdvisoryObservationSignature(bool present, string? format, string? keyId, string? signature) + { + Present = present; + Format = Validation.TrimToNull(format); + KeyId = Validation.TrimToNull(keyId); + Signature = Validation.TrimToNull(signature); + } + + public bool Present { get; } + + public string? Format { get; } + + public string? KeyId { get; } + + public string? Signature { get; } +} + +public sealed record AdvisoryObservationUpstream +{ + public AdvisoryObservationUpstream( + string upstreamId, + string? documentVersion, + DateTimeOffset fetchedAt, + DateTimeOffset receivedAt, + string contentHash, + AdvisoryObservationSignature signature, + ImmutableDictionary? metadata = null) + { + UpstreamId = Validation.EnsureNotNullOrWhiteSpace(upstreamId, nameof(upstreamId)); + DocumentVersion = Validation.TrimToNull(documentVersion); + FetchedAt = fetchedAt.ToUniversalTime(); + ReceivedAt = receivedAt.ToUniversalTime(); + ContentHash = Validation.EnsureNotNullOrWhiteSpace(contentHash, nameof(contentHash)); + Signature = signature ?? throw new ArgumentNullException(nameof(signature)); + Metadata = NormalizeMetadata(metadata); + } + + public string UpstreamId { get; } + + public string? DocumentVersion { get; } + + public DateTimeOffset FetchedAt { get; } + + public DateTimeOffset ReceivedAt { get; } + + public string ContentHash { get; } + + public AdvisoryObservationSignature Signature { get; } + + public ImmutableDictionary Metadata { get; } + + private static ImmutableDictionary NormalizeMetadata(ImmutableDictionary? metadata) + { + if (metadata is null || metadata.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var pair in metadata) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + builder[pair.Key.Trim()] = pair.Value; + } + + return builder.ToImmutable(); + } +} + +public sealed record AdvisoryObservationContent +{ + public AdvisoryObservationContent( + string format, + string? specVersion, + JsonNode raw, + ImmutableDictionary? metadata = null) + { + Format = Validation.EnsureNotNullOrWhiteSpace(format, nameof(format)); + SpecVersion = Validation.TrimToNull(specVersion); + Raw = raw?.DeepClone() ?? throw new ArgumentNullException(nameof(raw)); + Metadata = NormalizeMetadata(metadata); + } + + public string Format { get; } + + public string? SpecVersion { get; } + + public JsonNode Raw { get; } + + public ImmutableDictionary Metadata { get; } + + private static ImmutableDictionary NormalizeMetadata(ImmutableDictionary? metadata) + { + if (metadata is null || metadata.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var pair in metadata) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + builder[pair.Key.Trim()] = pair.Value; + } + + return builder.ToImmutable(); + } +} + +public sealed record AdvisoryObservationReference +{ + public AdvisoryObservationReference(string type, string url) + { + Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type)).ToLowerInvariant(); + Url = Validation.EnsureNotNullOrWhiteSpace(url, nameof(url)); + } + + public string Type { get; } + + public string Url { get; } +} + +public sealed record AdvisoryObservationLinkset +{ + public AdvisoryObservationLinkset( + IEnumerable? aliases, + IEnumerable? purls, + IEnumerable? cpes, + IEnumerable? references) + { + Aliases = NormalizeStringSet(aliases, toLower: true); + Purls = NormalizeStringSet(purls); + Cpes = NormalizeStringSet(cpes); + References = NormalizeReferences(references); + } + + public ImmutableArray Aliases { get; } + + public ImmutableArray Purls { get; } + + public ImmutableArray Cpes { get; } + + public ImmutableArray References { get; } + + private static ImmutableArray NormalizeStringSet(IEnumerable? values, bool toLower = false) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + var list = new List(); + foreach (var value in values) + { + var trimmed = Validation.TrimToNull(value); + if (trimmed is null) + { + continue; + } + + list.Add(toLower ? trimmed.ToLowerInvariant() : trimmed); + } + + return list + .Distinct(StringComparer.Ordinal) + .OrderBy(static v => v, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeReferences(IEnumerable? references) + { + if (references is null) + { + return ImmutableArray.Empty; + } + + return references + .Where(static reference => reference is not null) + .Distinct() + .OrderBy(static reference => reference.Type, StringComparer.Ordinal) + .ThenBy(static reference => reference.Url, StringComparer.Ordinal) + .ToImmutableArray(); + } +} diff --git a/src/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj b/src/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj index 1d9208cf..5d5e6d1c 100644 --- a/src/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj +++ b/src/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj @@ -1,12 +1,12 @@ - - - net10.0 - preview - enable - enable - true - - - - - + + + net10.0 + preview + enable + enable + true + + + + + diff --git a/src/StellaOps.Concelier.RawModels.Tests/StellaOps.Concelier.RawModels.Tests.csproj b/src/StellaOps.Concelier.RawModels.Tests/StellaOps.Concelier.RawModels.Tests.csproj new file mode 100644 index 00000000..b9185eb7 --- /dev/null +++ b/src/StellaOps.Concelier.RawModels.Tests/StellaOps.Concelier.RawModels.Tests.csproj @@ -0,0 +1,26 @@ + + + net10.0 + preview + enable + enable + true + Exe + false + false + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Concelier.RawModels.Tests/UnitTest1.cs b/src/StellaOps.Concelier.RawModels.Tests/UnitTest1.cs new file mode 100644 index 00000000..72a58b2e --- /dev/null +++ b/src/StellaOps.Concelier.RawModels.Tests/UnitTest1.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Concelier.RawModels.Tests; + +public class UnitTest1 +{ + [Fact] + public void Test1() + { + + } +} diff --git a/src/StellaOps.Concelier.RawModels.Tests/xunit.runner.json b/src/StellaOps.Concelier.RawModels.Tests/xunit.runner.json new file mode 100644 index 00000000..86c7ea05 --- /dev/null +++ b/src/StellaOps.Concelier.RawModels.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/src/StellaOps.Concelier.RawModels/AdvisoryRawDocument.cs b/src/StellaOps.Concelier.RawModels/AdvisoryRawDocument.cs new file mode 100644 index 00000000..a86af2e3 --- /dev/null +++ b/src/StellaOps.Concelier.RawModels/AdvisoryRawDocument.cs @@ -0,0 +1,60 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.RawModels; + +public sealed record AdvisoryRawDocument( + [property: JsonPropertyName("tenant")] string Tenant, + [property: JsonPropertyName("source")] RawSourceMetadata Source, + [property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream, + [property: JsonPropertyName("content")] RawContent Content, + [property: JsonPropertyName("identifiers")] RawIdentifiers Identifiers, + [property: JsonPropertyName("linkset")] RawLinkset Linkset, + [property: JsonPropertyName("supersedes")] string? Supersedes = null) +{ + public AdvisoryRawDocument WithSupersedes(string supersedes) + => this with { Supersedes = supersedes }; +} + +public sealed record RawSourceMetadata( + [property: JsonPropertyName("vendor")] string Vendor, + [property: JsonPropertyName("connector")] string Connector, + [property: JsonPropertyName("version")] string ConnectorVersion, + [property: JsonPropertyName("stream")] string? Stream = null); + +public sealed record RawUpstreamMetadata( + [property: JsonPropertyName("upstream_id")] string UpstreamId, + [property: JsonPropertyName("document_version")] string? DocumentVersion, + [property: JsonPropertyName("retrieved_at")] DateTimeOffset RetrievedAt, + [property: JsonPropertyName("content_hash")] string ContentHash, + [property: JsonPropertyName("signature")] RawSignatureMetadata Signature, + [property: JsonPropertyName("provenance")] ImmutableDictionary Provenance); + +public sealed record RawSignatureMetadata( + [property: JsonPropertyName("present")] bool Present, + [property: JsonPropertyName("format")] string? Format = null, + [property: JsonPropertyName("key_id")] string? KeyId = null, + [property: JsonPropertyName("sig")] string? Signature = null, + [property: JsonPropertyName("certificate")] string? Certificate = null, + [property: JsonPropertyName("digest")] string? Digest = null); + +public sealed record RawContent( + [property: JsonPropertyName("format")] string Format, + [property: JsonPropertyName("spec_version")] string? SpecVersion, + [property: JsonPropertyName("raw")] JsonElement Raw, + [property: JsonPropertyName("encoding")] string? Encoding = null); + +public sealed record RawIdentifiers( + [property: JsonPropertyName("aliases")] ImmutableArray Aliases, + [property: JsonPropertyName("primary")] string PrimaryId); + +public sealed record RawLinkset( + [property: JsonPropertyName("purls")] ImmutableArray PackageUrls, + [property: JsonPropertyName("cpes")] ImmutableArray Cpes, + [property: JsonPropertyName("references")] ImmutableArray References, + [property: JsonPropertyName("notes")] ImmutableDictionary Notes); + +public sealed record RawReference( + [property: JsonPropertyName("type")] string Type, + [property: JsonPropertyName("url")] string Url, + [property: JsonPropertyName("source")] string? Source = null); diff --git a/src/StellaOps.Concelier.RawModels/Class1.cs b/src/StellaOps.Concelier.RawModels/Class1.cs new file mode 100644 index 00000000..e55eda28 --- /dev/null +++ b/src/StellaOps.Concelier.RawModels/Class1.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Concelier.RawModels; + +public class Class1 +{ + +} diff --git a/src/StellaOps.Concelier.RawModels/JsonElementExtensions.cs b/src/StellaOps.Concelier.RawModels/JsonElementExtensions.cs new file mode 100644 index 00000000..eed6909f --- /dev/null +++ b/src/StellaOps.Concelier.RawModels/JsonElementExtensions.cs @@ -0,0 +1,12 @@ +using System.Text.Json; + +namespace StellaOps.Concelier.RawModels; + +internal static class JsonElementExtensions +{ + public static JsonElement CloneElement(this JsonElement element) + { + using var document = JsonDocument.Parse(element.GetRawText()); + return document.RootElement.Clone(); + } +} diff --git a/src/StellaOps.Concelier.RawModels/RawDocumentFactory.cs b/src/StellaOps.Concelier.RawModels/RawDocumentFactory.cs new file mode 100644 index 00000000..1932b495 --- /dev/null +++ b/src/StellaOps.Concelier.RawModels/RawDocumentFactory.cs @@ -0,0 +1 @@ +using System.Collections.Immutable;\nusing System.Text.Json;\n\nnamespace StellaOps.Concelier.RawModels;\n\npublic static class RawDocumentFactory\n{\n public static AdvisoryRawDocument CreateAdvisory(\n string tenant,\n RawSourceMetadata source,\n RawUpstreamMetadata upstream,\n RawContent content,\n RawIdentifiers identifiers,\n RawLinkset linkset,\n string? supersedes = null)\n {\n var clonedContent = content with { Raw = Clone(content.Raw) };\n return new AdvisoryRawDocument(tenant, source, upstream, clonedContent, identifiers, linkset, supersedes);\n }\n\n public static VexRawDocument CreateVex(\n string tenant,\n RawSourceMetadata source,\n RawUpstreamMetadata upstream,\n RawContent content,\n RawLinkset linkset,\n ImmutableArray statements,\n string? supersedes = null)\n {\n var clonedContent = content with { Raw = Clone(content.Raw) };\n return new VexRawDocument(tenant, source, upstream, clonedContent, linkset, statements, supersedes);\n }\n\n private static JsonElement Clone(JsonElement element)\n {\n using var document = JsonDocument.Parse(element.GetRawText());\n return document.RootElement.Clone();\n }\n}\n diff --git a/src/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj b/src/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj new file mode 100644 index 00000000..08b6b3a8 --- /dev/null +++ b/src/StellaOps.Concelier.RawModels/StellaOps.Concelier.RawModels.csproj @@ -0,0 +1,12 @@ + + + net10.0 + preview + enable + enable + true + + + + + diff --git a/src/StellaOps.Concelier.RawModels/VexRawDocument.cs b/src/StellaOps.Concelier.RawModels/VexRawDocument.cs new file mode 100644 index 00000000..3755a3ac --- /dev/null +++ b/src/StellaOps.Concelier.RawModels/VexRawDocument.cs @@ -0,0 +1,24 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.RawModels; + +public sealed record VexRawDocument( + [property: JsonPropertyName("tenant")] string Tenant, + [property: JsonPropertyName("source")] RawSourceMetadata Source, + [property: JsonPropertyName("upstream")] RawUpstreamMetadata Upstream, + [property: JsonPropertyName("content")] RawContent Content, + [property: JsonPropertyName("linkset")] RawLinkset Linkset, + [property: JsonPropertyName("statements")] ImmutableArray Statements, + [property: JsonPropertyName("supersedes")] string? Supersedes = null) +{ + public VexRawDocument WithSupersedes(string supersedes) + => this with { Supersedes = supersedes }; +} + +public sealed record VexStatementSummary( + [property: JsonPropertyName("advisory_ids")] ImmutableArray AdvisoryIds, + [property: JsonPropertyName("products")] ImmutableArray Products, + [property: JsonPropertyName("statuses")] ImmutableArray Statuses, + [property: JsonPropertyName("justification")] string? Justification = null, + [property: JsonPropertyName("impact")] string? Impact = null); diff --git a/src/StellaOps.Concelier.Storage.Mongo.Tests/Observations/AdvisoryObservationDocumentFactoryTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/Observations/AdvisoryObservationDocumentFactoryTests.cs new file mode 100644 index 00000000..bbd4d17a --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/Observations/AdvisoryObservationDocumentFactoryTests.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using StellaOps.Concelier.Storage.Mongo.Observations; +using Xunit; + +namespace StellaOps.Concelier.Storage.Mongo.Tests.Observations; + +public sealed class AdvisoryObservationDocumentFactoryTests +{ + [Fact] + public void ToModel_MapsDocumentToModel() + { + var document = new AdvisoryObservationDocument + { + Id = "tenant-a:obs-1", + Tenant = "tenant-a", + CreatedAt = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc), + Source = new AdvisoryObservationSourceDocument + { + Vendor = "vendor", + Stream = "stream", + Api = "https://api.example" + }, + Upstream = new AdvisoryObservationUpstreamDocument + { + UpstreamId = "CVE-2025-1234", + DocumentVersion = "1", + FetchedAt = DateTime.SpecifyKind(DateTime.UtcNow.AddMinutes(-1), DateTimeKind.Utc), + ReceivedAt = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc), + ContentHash = "sha256:abc", + Signature = new AdvisoryObservationSignatureDocument + { + Present = true, + Format = "pgp", + KeyId = "key", + Signature = "signature" + } + }, + Content = new AdvisoryObservationContentDocument + { + Format = "CSAF", + SpecVersion = "2.0", + Raw = BsonDocument.Parse("{\"example\":true}") + }, + Linkset = new AdvisoryObservationLinksetDocument + { + Aliases = new List { "CVE-2025-1234" }, + Purls = new List { "pkg:generic/foo@1.0.0" }, + Cpes = new List { "cpe:/a:vendor:product:1" }, + References = new List + { + new() { Type = "advisory", Url = "https://example.com" } + } + } + }; + + var observation = AdvisoryObservationDocumentFactory.ToModel(document); + + Assert.Equal("tenant-a:obs-1", observation.ObservationId); + Assert.Equal("tenant-a", observation.Tenant); + Assert.Equal("CVE-2025-1234", observation.Upstream.UpstreamId); + Assert.Contains("pkg:generic/foo@1.0.0", observation.Linkset.Purls); + Assert.Equal("CSAF", observation.Content.Format); + Assert.True(observation.Content.Raw?["example"]?.GetValue()); + Assert.Equal("advisory", observation.Linkset.References[0].Type); + } +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs index 2e7ccba4..1f1262a0 100644 --- a/src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs @@ -31,6 +31,7 @@ public sealed class MongoBootstrapper MongoStorageDefaults.Collections.ChangeHistory, MongoStorageDefaults.Collections.AdvisoryStatements, MongoStorageDefaults.Collections.AdvisoryConflicts, + MongoStorageDefaults.Collections.AdvisoryObservations, MongoStorageDefaults.Collections.Locks, MongoStorageDefaults.Collections.Jobs, MongoStorageDefaults.Collections.Migrations, @@ -79,6 +80,7 @@ public sealed class MongoBootstrapper EnsurePsirtFlagIndexesAsync(cancellationToken), EnsureAdvisoryStatementIndexesAsync(cancellationToken), EnsureAdvisoryConflictIndexesAsync(cancellationToken), + EnsureObservationIndexesAsync(cancellationToken), EnsureChangeHistoryIndexesAsync(cancellationToken), EnsureGridFsIndexesAsync(cancellationToken)).ConfigureAwait(false); @@ -246,14 +248,45 @@ public sealed class MongoBootstrapper { var collection = _database.GetCollection(MongoStorageDefaults.Collections.Reference); var indexes = new List> - { - new( - Builders.IndexKeys.Ascending("url"), - new CreateIndexOptions { Name = "reference_url" }), - new( - Builders.IndexKeys.Ascending("advisoryId"), - new CreateIndexOptions { Name = "reference_advisoryId" }), - }; + { + new( + Builders.IndexKeys.Ascending("url"), + new CreateIndexOptions { Name = "reference_url" }), + new( + Builders.IndexKeys.Ascending("advisoryId"), + new CreateIndexOptions { Name = "reference_advisoryId" }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureObservationIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.AdvisoryObservations); + var indexes = new List> + { + new( + Builders.IndexKeys + .Ascending("tenant") + .Ascending("upstream.upstream_id") + .Ascending("upstream.document_version"), + new CreateIndexOptions { Name = "advisory_obs_tenant_upstream", Unique = false }), + new( + Builders.IndexKeys + .Ascending("tenant") + .Ascending("linkset.aliases"), + new CreateIndexOptions { Name = "advisory_obs_tenant_aliases" }), + new( + Builders.IndexKeys + .Ascending("tenant") + .Ascending("linkset.purls"), + new CreateIndexOptions { Name = "advisory_obs_tenant_purls" }), + new( + Builders.IndexKeys + .Ascending("tenant") + .Descending("createdAt"), + new CreateIndexOptions { Name = "advisory_obs_tenant_createdAt" }) + }; return collection.Indexes.CreateManyAsync(indexes, cancellationToken); } diff --git a/src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs index f8f6492c..6e3534d0 100644 --- a/src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs @@ -26,5 +26,6 @@ public static class MongoStorageDefaults public const string ChangeHistory = "source_change_history"; public const string AdvisoryStatements = "advisory_statements"; public const string AdvisoryConflicts = "advisory_conflicts"; + public const string AdvisoryObservations = "advisory_observations"; } } diff --git a/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationDocument.cs new file mode 100644 index 00000000..8a7db37e --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationDocument.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Concelier.Storage.Mongo.Observations; + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("tenant")] + public string Tenant { get; set; } = string.Empty; + + [BsonElement("source")] + public AdvisoryObservationSourceDocument Source { get; set; } = new(); + + [BsonElement("upstream")] + public AdvisoryObservationUpstreamDocument Upstream { get; set; } = new(); + + [BsonElement("content")] + public AdvisoryObservationContentDocument Content { get; set; } = new(); + + [BsonElement("linkset")] + public AdvisoryObservationLinksetDocument Linkset { get; set; } = new(); + + [BsonElement("createdAt")] + public DateTime CreatedAt { get; set; } + = DateTime.UtcNow; + + [BsonElement("attributes")] + [BsonIgnoreIfNull] + public Dictionary? Attributes { get; set; } + = new(StringComparer.Ordinal); +} + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationSourceDocument +{ + [BsonElement("vendor")] + public string Vendor { get; set; } = string.Empty; + + [BsonElement("stream")] + public string Stream { get; set; } = string.Empty; + + [BsonElement("api")] + public string Api { get; set; } = string.Empty; + + [BsonElement("collectorVersion")] + [BsonIgnoreIfNull] + public string? CollectorVersion { get; set; } + = null; +} + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationUpstreamDocument +{ + [BsonElement("upstream_id")] + public string UpstreamId { get; set; } = string.Empty; + + [BsonElement("document_version")] + [BsonIgnoreIfNull] + public string? DocumentVersion { get; set; } + = null; + + [BsonElement("fetchedAt")] + public DateTime FetchedAt { get; set; } + = DateTime.UtcNow; + + [BsonElement("receivedAt")] + public DateTime ReceivedAt { get; set; } + = DateTime.UtcNow; + + [BsonElement("contentHash")] + public string ContentHash { get; set; } = string.Empty; + + [BsonElement("signature")] + public AdvisoryObservationSignatureDocument Signature { get; set; } = new(); + + [BsonElement("metadata")] + [BsonIgnoreIfNull] + public Dictionary? Metadata { get; set; } + = new(StringComparer.Ordinal); +} + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationSignatureDocument +{ + [BsonElement("present")] + public bool Present { get; set; } + = false; + + [BsonElement("format")] + [BsonIgnoreIfNull] + public string? Format { get; set; } + = null; + + [BsonElement("keyId")] + [BsonIgnoreIfNull] + public string? KeyId { get; set; } + = null; + + [BsonElement("signature")] + [BsonIgnoreIfNull] + public string? Signature { get; set; } + = null; +} + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationContentDocument +{ + [BsonElement("format")] + public string Format { get; set; } = string.Empty; + + [BsonElement("specVersion")] + [BsonIgnoreIfNull] + public string? SpecVersion { get; set; } + = null; + + [BsonElement("raw")] + public BsonDocument Raw { get; set; } = new(); + + [BsonElement("metadata")] + [BsonIgnoreIfNull] + public Dictionary? Metadata { get; set; } + = new(StringComparer.Ordinal); +} + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationLinksetDocument +{ + [BsonElement("aliases")] + [BsonIgnoreIfNull] + public List? Aliases { get; set; } + = new(); + + [BsonElement("purls")] + [BsonIgnoreIfNull] + public List? Purls { get; set; } + = new(); + + [BsonElement("cpes")] + [BsonIgnoreIfNull] + public List? Cpes { get; set; } + = new(); + + [BsonElement("references")] + [BsonIgnoreIfNull] + public List? References { get; set; } + = new(); +} + +[BsonIgnoreExtraElements] +public sealed class AdvisoryObservationReferenceDocument +{ + [BsonElement("type")] + public string Type { get; set; } = string.Empty; + + [BsonElement("url")] + public string Url { get; set; } = string.Empty; +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationDocumentFactory.cs b/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationDocumentFactory.cs new file mode 100644 index 00000000..03886a9b --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationDocumentFactory.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Nodes; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Concelier.Models.Observations; + +namespace StellaOps.Concelier.Storage.Mongo.Observations; + +internal static class AdvisoryObservationDocumentFactory +{ + private static readonly JsonWriterSettings JsonSettings = new() { OutputMode = JsonOutputMode.RelaxedExtendedJson }; + + public static AdvisoryObservation ToModel(AdvisoryObservationDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + var rawNode = ParseJsonNode(document.Content.Raw); + var attributes = ToImmutable(document.Attributes); + var contentMetadata = ToImmutable(document.Content.Metadata); + var upstreamMetadata = ToImmutable(document.Upstream.Metadata); + + var observation = new AdvisoryObservation( + document.Id, + document.Tenant, + new AdvisoryObservationSource( + document.Source.Vendor, + document.Source.Stream, + document.Source.Api, + document.Source.CollectorVersion), + new AdvisoryObservationUpstream( + document.Upstream.UpstreamId, + document.Upstream.DocumentVersion, + DateTime.SpecifyKind(document.Upstream.FetchedAt, DateTimeKind.Utc), + DateTime.SpecifyKind(document.Upstream.ReceivedAt, DateTimeKind.Utc), + document.Upstream.ContentHash, + new AdvisoryObservationSignature( + document.Upstream.Signature.Present, + document.Upstream.Signature.Format, + document.Upstream.Signature.KeyId, + document.Upstream.Signature.Signature), + upstreamMetadata), + new AdvisoryObservationContent( + document.Content.Format, + document.Content.SpecVersion, + rawNode, + contentMetadata), + new AdvisoryObservationLinkset( + document.Linkset.Aliases ?? Enumerable.Empty(), + document.Linkset.Purls ?? Enumerable.Empty(), + document.Linkset.Cpes ?? Enumerable.Empty(), + document.Linkset.References?.Select(reference => new AdvisoryObservationReference(reference.Type, reference.Url))), + DateTime.SpecifyKind(document.CreatedAt, DateTimeKind.Utc), + attributes); + + return observation; + } + + private static JsonNode ParseJsonNode(BsonDocument raw) + { + if (raw is null || raw.ElementCount == 0) + { + return JsonNode.Parse("{}")!; + } + + var json = raw.ToJson(JsonSettings); + return JsonNode.Parse(json)!; + } + + private static ImmutableDictionary ToImmutable(Dictionary? values) + { + if (values is null || values.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var pair in values) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + builder[pair.Key.Trim()] = pair.Value; + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationStore.cs new file mode 100644 index 00000000..e02b0f32 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Observations/AdvisoryObservationStore.cs @@ -0,0 +1,57 @@ +using MongoDB.Driver; +using StellaOps.Concelier.Models.Observations; + +namespace StellaOps.Concelier.Storage.Mongo.Observations; + +internal sealed class AdvisoryObservationStore : IAdvisoryObservationStore +{ + private readonly IMongoCollection collection; + + public AdvisoryObservationStore(IMongoCollection collection) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + } + + public async Task> ListByTenantAsync(string tenant, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tenant); + + var filter = Builders.Filter.Eq(document => document.Tenant, tenant.ToLowerInvariant()); + var documents = await collection + .Find(filter) + .SortByDescending(document => document.CreatedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents.Select(AdvisoryObservationDocumentFactory.ToModel).ToArray(); + } + + public async Task> FindByAliasesAsync(string tenant, IEnumerable aliases, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tenant); + ArgumentNullException.ThrowIfNull(aliases); + + var normalizedTenant = tenant.ToLowerInvariant(); + var normalizedAliases = aliases + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Select(static alias => alias.Trim().ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + if (normalizedAliases.Length == 0) + { + return Array.Empty(); + } + + var filter = Builders.Filter.And( + Builders.Filter.Eq(document => document.Tenant, normalizedTenant), + Builders.Filter.In("linkset.aliases", normalizedAliases)); + + var documents = await collection + .Find(filter) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents.Select(AdvisoryObservationDocumentFactory.ToModel).ToArray(); + } +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Observations/IAdvisoryObservationStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Observations/IAdvisoryObservationStore.cs new file mode 100644 index 00000000..66241419 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Observations/IAdvisoryObservationStore.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using StellaOps.Concelier.Models.Observations; + +namespace StellaOps.Concelier.Storage.Mongo.Observations; + +public interface IAdvisoryObservationStore +{ + Task> ListByTenantAsync(string tenant, CancellationToken cancellationToken); + + Task> FindByAliasesAsync(string tenant, IEnumerable aliases, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs index d726761a..4bcad733 100644 --- a/src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs @@ -16,11 +16,12 @@ using StellaOps.Concelier.Storage.Mongo.PsirtFlags; using StellaOps.Concelier.Storage.Mongo.Statements; using StellaOps.Concelier.Storage.Mongo.Events; using StellaOps.Concelier.Core.Events; -using StellaOps.Concelier.Storage.Mongo.Migrations; - -namespace StellaOps.Concelier.Storage.Mongo; - -public static class ServiceCollectionExtensions +using StellaOps.Concelier.Storage.Mongo.Migrations; +using StellaOps.Concelier.Storage.Mongo.Observations; + +namespace StellaOps.Concelier.Storage.Mongo; + +public static class ServiceCollectionExtensions { public static IServiceCollection AddMongoStorage(this IServiceCollection services, Action configureOptions) { @@ -66,15 +67,16 @@ public static class ServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.TryAddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.TryAddSingleton(); services.AddSingleton>(static sp => { @@ -82,16 +84,22 @@ public static class ServiceCollectionExtensions return database.GetCollection(MongoStorageDefaults.Collections.Jobs); }); - services.AddSingleton>(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(MongoStorageDefaults.Collections.Locks); - }); - - services.AddHostedService(); - - services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton>(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(MongoStorageDefaults.Collections.Locks); + }); + + services.AddSingleton>(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(MongoStorageDefaults.Collections.AdvisoryObservations); + }); + + services.AddHostedService(); + + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj b/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj index 57200b41..2cf31682 100644 --- a/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj +++ b/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - + + + net10.0 + preview + enable + enable + true + + - - - - - - - - + + + + + + + + diff --git a/src/StellaOps.Concelier.Storage.Mongo/TASKS.md b/src/StellaOps.Concelier.Storage.Mongo/TASKS.md index 28a0a399..b6a8caa4 100644 --- a/src/StellaOps.Concelier.Storage.Mongo/TASKS.md +++ b/src/StellaOps.Concelier.Storage.Mongo/TASKS.md @@ -3,9 +3,13 @@ | ID | Status | Owner(s) | Depends on | Notes | |---|---|---|---|---| | CONCELIER-STORE-AOC-19-001 `advisory_raw schema validator` | TODO | Concelier Storage Guild | Mongo cluster ops sign-off | Author MongoDB JSON schema enforcing required fields (`source`, `upstream`, `content`, `linkset`, `tenant`) and forbidding normalized/severity fields. Include migration toggles for staged rollout. | +> Docs alignment (2025-10-26): Validator expectations + deployment steps documented in `docs/deploy/containers.md` §1. | CONCELIER-STORE-AOC-19-002 `idempotency unique index` | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-001 | Create compound unique index on `(source.vendor, upstream.upstream_id, upstream.content_hash, tenant)` with backfill script verifying existing data, and document offline validator bootstrap. | +> Docs alignment (2025-10-26): Idempotency contract + supersedes metrics in `docs/ingestion/aggregation-only-contract.md` §7 and observability guide. | CONCELIER-STORE-AOC-19-003 `append-only supersedes migration` | TODO | Concelier Storage Guild | CONCELIER-STORE-AOC-19-002 | Introduce migration that freezes legacy `advisories` writes, copies data into `_backup_*`, and backfills supersedes pointers for raw revisions. Provide rollback plan. | +> Docs alignment (2025-10-26): Rollback guidance added to `docs/deploy/containers.md` §6. | CONCELIER-STORE-AOC-19-004 `validator deployment playbook` | TODO | Concelier Storage Guild, DevOps Guild | CONCELIER-STORE-AOC-19-001 | Update `MIGRATIONS.md` and Offline Kit docs to cover enabling validators, rolling restarts, and validator smoke tests for air-gapped installs. | +> Docs alignment (2025-10-26): Offline kit requirements documented in `docs/deploy/containers.md` §5. ## Policy Engine v2 diff --git a/src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj b/src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj index 64eb15ce..74beb9ac 100644 --- a/src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj +++ b/src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj @@ -1,4 +1,4 @@ - + net10.0 enable @@ -6,15 +6,15 @@ true false - - - - - all - - - - - - - + + + + + all + + + + + + + diff --git a/src/StellaOps.Concelier.WebService/Program.cs b/src/StellaOps.Concelier.WebService/Program.cs index 9e145c47..748eaf64 100644 --- a/src/StellaOps.Concelier.WebService/Program.cs +++ b/src/StellaOps.Concelier.WebService/Program.cs @@ -33,6 +33,7 @@ using StellaOps.Configuration; using StellaOps.Auth.Abstractions; using StellaOps.Auth.Client; using StellaOps.Auth.ServerIntegration; +using StellaOps.Aoc; var builder = WebApplication.CreateBuilder(args); @@ -83,6 +84,7 @@ builder.Services.AddJobScheduler(); builder.Services.AddBuiltInConcelierJobs(); builder.Services.AddSingleton(sp => new ServiceStatus(sp.GetRequiredService())); +builder.Services.AddAocGuard(); var authorityConfigured = concelierOptions.Authority is { Enabled: true }; diff --git a/src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj b/src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj index 7fedfa8b..fe134f86 100644 --- a/src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj +++ b/src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj @@ -1,29 +1,29 @@ - - - net10.0 - preview - enable - enable - true - StellaOps.Concelier.WebService - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + StellaOps.Concelier.WebService + + + + + + + + + + + + + + + + + + @@ -32,5 +32,6 @@ + diff --git a/src/StellaOps.Concelier.WebService/TASKS.md b/src/StellaOps.Concelier.WebService/TASKS.md index 541fcb72..2af915aa 100644 --- a/src/StellaOps.Concelier.WebService/TASKS.md +++ b/src/StellaOps.Concelier.WebService/TASKS.md @@ -3,9 +3,13 @@ | ID | Status | Owner(s) | Depends on | Notes | |---|---|---|---|---| | CONCELIER-WEB-AOC-19-001 `Raw ingestion endpoints` | TODO | Concelier WebService Guild | CONCELIER-CORE-AOC-19-001, CONCELIER-STORE-AOC-19-001 | Implement `POST /ingest/advisory`, `GET /advisories/raw*`, and `POST /aoc/verify` minimal API endpoints. Enforce new Authority scopes, inject tenant claims, and surface `AOCWriteGuard` to repository calls. | +> Docs alignment (2025-10-26): Endpoint expectations + scope requirements detailed in `docs/ingestion/aggregation-only-contract.md` and `docs/security/authority-scopes.md`. | CONCELIER-WEB-AOC-19-002 `AOC observability` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-WEB-AOC-19-001 | Emit `ingestion_write_total`, `aoc_violation_total`, latency histograms, and tracing spans (`ingest.fetch/transform/write`, `aoc.guard`). Wire structured logging to include tenant, source vendor, upstream id, and content hash. | +> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`. | CONCELIER-WEB-AOC-19-003 `Schema/guard unit tests` | TODO | QA Guild | CONCELIER-WEB-AOC-19-001 | Add unit tests covering schema validation failures, forbidden field rejections (`ERR_AOC_001/002/006/007`), idempotent upserts, and supersedes chains using deterministic fixtures. | +> Docs alignment (2025-10-26): Guard rules + error codes documented in AOC reference §5 and CLI guide. | CONCELIER-WEB-AOC-19-004 `End-to-end ingest verification` | TODO | Concelier WebService Guild, QA Guild | CONCELIER-WEB-AOC-19-003, CONCELIER-CORE-AOC-19-002 | Create integration tests ingesting large advisory batches (cold/warm) validating linkset enrichment, metrics emission, and reproducible outputs. Capture load-test scripts + doc notes for Offline Kit dry runs. | +> Docs alignment (2025-10-26): Offline verification workflow referenced in `docs/deploy/containers.md` §5. ## Policy Engine v2 @@ -25,8 +29,6 @@ | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| CONCELIER-GRAPH-21-003 `SBOM projection API` | TODO | Concelier WebService Guild, Cartographer Guild | CONCELIER-GRAPH-21-001 | Expose normalized SBOM projection endpoint (`/sboms/{id}/projections/graph`) with pagination, tenant guard, and versioning metadata for Cartographer builds. | -| CONCELIER-GRAPH-21-004 `Entry point registry` | TODO | Concelier WebService Guild | CONCELIER-GRAPH-21-003, SBOM-SERVICE-21-001 | Provide entrypoint/service node lookup API for Cartographer path relevance (configurable overrides, tagging) and document contract. | ## Link-Not-Merge v1 @@ -41,13 +43,19 @@ | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| | CONCELIER-GRAPH-24-101 `Advisory summary API` | TODO | Concelier WebService Guild | CONCELIER-GRAPH-24-001 | Expose `/advisories/summary` returning raw linkset/observation metadata for overlay services; no derived severity or fix hints. | -| CONCELIER-GRAPH-24-102 `Evidence batch API` | TODO | Concelier WebService Guild | CONCELIER-LNM-21-201 | Add batch fetch for advisory observations/linksets keyed by component sets to feed Graph overlay tooltips efficiently. | +| CONCELIER-GRAPH-28-102 `Evidence batch API` | TODO | Concelier WebService Guild | CONCELIER-LNM-21-201 | Add batch fetch for advisory observations/linksets keyed by component sets to feed Graph overlay tooltips efficiently. | + +## VEX Lens (Sprint 30) + +| ID | Status | Owner(s) | Depends on | Notes | +|----|--------|----------|------------|-------| +| CONCELIER-VEXLENS-30-001 `Advisory rationale bridges` | TODO | Concelier WebService Guild, VEX Lens Guild | CONCELIER-VULN-29-001, VEXLENS-30-005 | Guarantee advisory key consistency and cross-links for consensus rationale; Label: VEX-Lens. | ## Vulnerability Explorer (Sprint 29) | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| CONCELIER-VULN-29-001 `Advisory key normalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Normalize advisory identifiers (CVE/GHSA/vendor) into canonical `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs. Include migration/backfill scripts. | +| CONCELIER-VULN-29-001 `Advisory key canonicalization` | TODO | Concelier WebService Guild, Data Integrity Guild | CONCELIER-LNM-21-001 | Canonicalize (lossless) advisory identifiers (CVE/GHSA/vendor) into `advisory_key`, persist `links[]`, expose raw payload snapshots for Explorer evidence tabs; AOC-compliant: no merge, no derived fields, no suppression. Include migration/backfill scripts. | | CONCELIER-VULN-29-002 `Evidence retrieval API` | TODO | Concelier WebService Guild | CONCELIER-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/advisories/{advisory_key}` returning raw advisory docs with provenance, filtering by tenant and source. | | CONCELIER-VULN-29-004 `Observability enhancements` | TODO | Concelier WebService Guild, Observability Guild | CONCELIER-VULN-29-001 | Instrument metrics/logs for advisory normalization (key collisions, withdrawn flags), emit events consumed by Vuln Explorer resolver. | diff --git a/src/StellaOps.Concelier.sln b/src/StellaOps.Concelier.sln index e6ea515e..ae427cc7 100644 --- a/src/StellaOps.Concelier.sln +++ b/src/StellaOps.Concelier.sln @@ -135,6 +135,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connect EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Nkcki.Tests", "StellaOps.Concelier.Connector.Ru.Nkcki.Tests\StellaOps.Concelier.Connector.Ru.Nkcki.Tests.csproj", "{461D4A58-3816-4737-B209-2D1F08B1F4DF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Aoc", "StellaOps.Aoc\StellaOps.Aoc.csproj", "{FB026A42-9B11-4926-9918-249A672DB879}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Aoc.Tests", "StellaOps.Aoc.Tests\StellaOps.Aoc.Tests.csproj", "{2983C054-33F1-49A3-BB6A-FC4737B2A510}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.RawModels", "StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj", "{E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.RawModels.Tests", "StellaOps.Concelier.RawModels.Tests\StellaOps.Concelier.RawModels.Tests.csproj", "{38E1D9C6-F396-46EF-9A44-B7CC473B3D50}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -937,6 +945,54 @@ Global {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x64.Build.0 = Release|Any CPU {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x86.ActiveCfg = Release|Any CPU {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x86.Build.0 = Release|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Debug|x64.ActiveCfg = Debug|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Debug|x64.Build.0 = Debug|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Debug|x86.ActiveCfg = Debug|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Debug|x86.Build.0 = Debug|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Release|Any CPU.Build.0 = Release|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Release|x64.ActiveCfg = Release|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Release|x64.Build.0 = Release|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Release|x86.ActiveCfg = Release|Any CPU + {FB026A42-9B11-4926-9918-249A672DB879}.Release|x86.Build.0 = Release|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Debug|x64.ActiveCfg = Debug|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Debug|x64.Build.0 = Debug|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Debug|x86.ActiveCfg = Debug|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Debug|x86.Build.0 = Debug|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Release|Any CPU.Build.0 = Release|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Release|x64.ActiveCfg = Release|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Release|x64.Build.0 = Release|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Release|x86.ActiveCfg = Release|Any CPU + {2983C054-33F1-49A3-BB6A-FC4737B2A510}.Release|x86.Build.0 = Release|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Debug|x64.ActiveCfg = Debug|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Debug|x64.Build.0 = Debug|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Debug|x86.ActiveCfg = Debug|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Debug|x86.Build.0 = Debug|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Release|Any CPU.Build.0 = Release|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Release|x64.ActiveCfg = Release|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Release|x64.Build.0 = Release|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Release|x86.ActiveCfg = Release|Any CPU + {E8A8FDFC-DACF-4F7B-AECC-3D1C59341DF9}.Release|x86.Build.0 = Release|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Debug|Any CPU.Build.0 = Debug|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Debug|x64.ActiveCfg = Debug|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Debug|x64.Build.0 = Debug|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Debug|x86.ActiveCfg = Debug|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Debug|x86.Build.0 = Debug|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Release|Any CPU.ActiveCfg = Release|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Release|Any CPU.Build.0 = Release|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Release|x64.ActiveCfg = Release|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Release|x64.Build.0 = Release|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Release|x86.ActiveCfg = Release|Any CPU + {38E1D9C6-F396-46EF-9A44-B7CC473B3D50}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/StellaOps.Configuration/StellaOps.Configuration.csproj b/src/StellaOps.Configuration/StellaOps.Configuration.csproj index bf641143..65eb799b 100644 --- a/src/StellaOps.Configuration/StellaOps.Configuration.csproj +++ b/src/StellaOps.Configuration/StellaOps.Configuration.csproj @@ -1,17 +1,17 @@ - - - - net10.0 - enable - enable - - + + + + net10.0 + enable + enable + + - - - - - + + + + + diff --git a/src/StellaOps.Cryptography.DependencyInjection/StellaOps.Cryptography.DependencyInjection.csproj b/src/StellaOps.Cryptography.DependencyInjection/StellaOps.Cryptography.DependencyInjection.csproj index 8d4e3e5d..10a52416 100644 --- a/src/StellaOps.Cryptography.DependencyInjection/StellaOps.Cryptography.DependencyInjection.csproj +++ b/src/StellaOps.Cryptography.DependencyInjection/StellaOps.Cryptography.DependencyInjection.csproj @@ -8,7 +8,7 @@ - - + + diff --git a/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj b/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj index f0180975..64aabe1f 100644 --- a/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj +++ b/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Cryptography.Tests/Audit/AuthEventRecordTests.cs b/src/StellaOps.Cryptography.Tests/Audit/AuthEventRecordTests.cs index b042cd8a..d090eef5 100644 --- a/src/StellaOps.Cryptography.Tests/Audit/AuthEventRecordTests.cs +++ b/src/StellaOps.Cryptography.Tests/Audit/AuthEventRecordTests.cs @@ -18,6 +18,7 @@ public class AuthEventRecordTests Assert.Empty(record.Scopes); Assert.NotNull(record.Properties); Assert.Empty(record.Properties); + Assert.False(record.Tenant.HasValue); } [Fact] diff --git a/src/StellaOps.Cryptography/Audit/AuthEventRecord.cs b/src/StellaOps.Cryptography/Audit/AuthEventRecord.cs index 9bfa1ddd..3636852a 100644 --- a/src/StellaOps.Cryptography/Audit/AuthEventRecord.cs +++ b/src/StellaOps.Cryptography/Audit/AuthEventRecord.cs @@ -43,6 +43,11 @@ public sealed record AuthEventRecord /// public AuthEventClient? Client { get; init; } + /// + /// Tenant identifier associated with the authenticated principal or client. + /// + public ClassifiedString Tenant { get; init; } = ClassifiedString.Empty; + /// /// Granted or requested scopes tied to the event. /// diff --git a/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj b/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj index 543198c5..a83f9a6a 100644 --- a/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj +++ b/src/StellaOps.Cryptography/StellaOps.Cryptography.csproj @@ -11,6 +11,6 @@ - + diff --git a/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj b/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj index 2be35349..679434be 100644 --- a/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj +++ b/src/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj @@ -1,14 +1,14 @@ - - - - net10.0 - enable - enable - - - - - - - + + + + net10.0 + enable + enable + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.WebService/StellaOps.EvidenceLocker.WebService.csproj b/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.WebService/StellaOps.EvidenceLocker.WebService.csproj index 15f9a74e..24d43152 100644 --- a/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.WebService/StellaOps.EvidenceLocker.WebService.csproj +++ b/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.WebService/StellaOps.EvidenceLocker.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.Worker/StellaOps.EvidenceLocker.Worker.csproj b/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.Worker/StellaOps.EvidenceLocker.Worker.csproj index 847895fd..ee851b91 100644 --- a/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.Worker/StellaOps.EvidenceLocker.Worker.csproj +++ b/src/StellaOps.EvidenceLocker/StellaOps.EvidenceLocker.Worker/StellaOps.EvidenceLocker.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.EvidenceLocker/TASKS.md b/src/StellaOps.EvidenceLocker/TASKS.md index cf085050..ec084597 100644 --- a/src/StellaOps.EvidenceLocker/TASKS.md +++ b/src/StellaOps.EvidenceLocker/TASKS.md @@ -17,3 +17,8 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | EVID-OBS-55-001 | TODO | Evidence Locker Guild, DevOps Guild | EVID-OBS-54-001, DEVOPS-OBS-55-001 | Implement incident mode hooks increasing retention window, capturing additional debug artefacts, and emitting activation/deactivation events to Timeline Indexer + Notifier. | Incident mode extends retention per config; activation events emitted; tests cover revert to baseline; runbook updated. | + +## Sprint 60 – Sealed Mode Portability +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| EVID-OBS-60-001 | TODO | Evidence Locker Guild | EVID-OBS-55-001, AIRGAP-CTL-56-002 | Deliver portable evidence export flow for sealed environments: generate sealed bundles with checksum manifest, redacted metadata, and offline verification script. Document air-gapped import/verify procedures. | Portable bundle tooling implemented; checksum/verify script passes; sealed-mode docs updated; tests cover tamper + re-import scenarios. | diff --git a/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj b/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj index ee590291..a7250c2a 100644 --- a/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj +++ b/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj @@ -8,8 +8,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj b/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj index 312a9f5a..9f1edfc8 100644 --- a/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj +++ b/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj @@ -7,9 +7,9 @@ true - - - + + + diff --git a/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj b/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj index 1222a9c0..5f8740e5 100644 --- a/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj +++ b/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj @@ -10,8 +10,8 @@ - - - + + + diff --git a/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj index fc88a048..c573f653 100644 --- a/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj index 96553da9..fc0027b7 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj index 036c860d..48287a90 100644 --- a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj index 623f5e8f..9ca8293a 100644 --- a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj index 6b331924..44376895 100644 --- a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj index 6048a328..528090b4 100644 --- a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj index fc88a048..c573f653 100644 --- a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj index 036c860d..48287a90 100644 --- a/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj index 036c860d..48287a90 100644 --- a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj @@ -12,8 +12,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj index f6efcd05..3fe0a543 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj index fc88a048..c573f653 100644 --- a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/src/StellaOps.Excititor.Core/TASKS.md b/src/StellaOps.Excititor.Core/TASKS.md index a3f5e680..bbc1483f 100644 --- a/src/StellaOps.Excititor.Core/TASKS.md +++ b/src/StellaOps.Excititor.Core/TASKS.md @@ -6,6 +6,7 @@ | EXCITITOR-CORE-AOC-19-002 `VEX linkset extraction` | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-001 | Implement deterministic extraction of advisory IDs, component PURLs, and references into `linkset`, capturing reconciled-from metadata for traceability. | | EXCITITOR-CORE-AOC-19-003 `Idempotent VEX raw upsert` | TODO | Excititor Core Guild | EXCITITOR-STORE-AOC-19-002 | Enforce `(vendor, upstreamId, contentHash, tenant)` uniqueness, generate supersedes chains, and ensure append-only versioning of raw VEX documents. | | EXCITITOR-CORE-AOC-19-004 `Remove ingestion consensus` | TODO | Excititor Core Guild | EXCITITOR-CORE-AOC-19-002, POLICY-AOC-19-003 | Excise consensus/merge/severity logic from Excititor ingestion paths, updating exports/tests to rely on Policy Engine materializations instead. | +| EXCITITOR-CORE-AOC-19-013 `Authority tenant scope smoke coverage` | TODO | Excititor Core Guild | AUTH-AOC-19-002 | Update Excititor smoke/e2e suites to seed tenant-aware Authority clients and ensure cross-tenant VEX ingestion is rejected. | Required for Authority docs (`AUTH-AOC-19-003`) sign-off; share results with Authority Core. | ## Policy Engine v2 diff --git a/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj b/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj index 983f00d5..d054a3d9 100644 --- a/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj +++ b/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj @@ -7,8 +7,8 @@ true - - + + diff --git a/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj b/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj index 8455f3fc..ba0a8da1 100644 --- a/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj +++ b/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj @@ -7,8 +7,8 @@ true - - + + diff --git a/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj b/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj index 8455f3fc..ba0a8da1 100644 --- a/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj +++ b/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj @@ -7,8 +7,8 @@ true - - + + diff --git a/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj b/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj index 8455f3fc..ba0a8da1 100644 --- a/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj +++ b/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj @@ -7,8 +7,8 @@ true - - + + diff --git a/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj b/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj index 2433659d..cb5db022 100644 --- a/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj +++ b/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj @@ -7,8 +7,8 @@ true - - + + diff --git a/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj b/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj index a2181f1e..4c3f02df 100644 --- a/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj +++ b/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj @@ -7,9 +7,9 @@ true - - - + + + diff --git a/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj b/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj index d8823043..874ab412 100644 --- a/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj +++ b/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj @@ -9,8 +9,8 @@ - - + + diff --git a/src/StellaOps.Excititor.WebService/Program.cs b/src/StellaOps.Excititor.WebService/Program.cs index 9a41e477..1540ae06 100644 --- a/src/StellaOps.Excititor.WebService/Program.cs +++ b/src/StellaOps.Excititor.WebService/Program.cs @@ -19,6 +19,7 @@ using StellaOps.Excititor.Storage.Mongo; using StellaOps.Excititor.WebService.Endpoints; using StellaOps.Excititor.WebService.Services; using StellaOps.Excititor.Core; +using StellaOps.Aoc; var builder = WebApplication.CreateBuilder(args); var configuration = builder.Configuration; @@ -28,12 +29,13 @@ services.AddOptions() .Bind(configuration.GetSection("Excititor:Storage:Mongo")) .ValidateOnStart(); -services.AddExcititorMongoStorage(); -services.AddCsafNormalizer(); -services.AddCycloneDxNormalizer(); -services.AddOpenVexNormalizer(); -services.AddSingleton(); -services.AddScoped(); +services.AddExcititorMongoStorage(); +services.AddCsafNormalizer(); +services.AddCycloneDxNormalizer(); +services.AddOpenVexNormalizer(); +services.AddSingleton(); +services.AddScoped(); +services.AddAocGuard(); services.AddVexExportEngine(); services.AddVexExportCacheServices(); services.AddVexAttestation(); diff --git a/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj b/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj index 17812bc5..5df0d48b 100644 --- a/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj +++ b/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj @@ -13,9 +13,10 @@ - - - - - - + + + + + + + diff --git a/src/StellaOps.Excititor.WebService/TASKS.md b/src/StellaOps.Excititor.WebService/TASKS.md index dcde0440..749569cf 100644 --- a/src/StellaOps.Excititor.WebService/TASKS.md +++ b/src/StellaOps.Excititor.WebService/TASKS.md @@ -3,9 +3,13 @@ | ID | Status | Owner(s) | Depends on | Notes | |---|---|---|---|---| | EXCITITOR-WEB-AOC-19-001 `Raw VEX ingestion APIs` | TODO | Excititor WebService Guild | EXCITITOR-CORE-AOC-19-001, EXCITITOR-STORE-AOC-19-001 | Implement `POST /ingest/vex`, `GET /vex/raw*`, and `POST /aoc/verify` endpoints. Enforce Authority scopes, tenant injection, and guard pipeline to ensure only immutable VEX facts are persisted. | +> Docs alignment (2025-10-26): See AOC reference §4–5 and authority scopes doc for required tokens/behaviour. | EXCITITOR-WEB-AOC-19-002 `AOC observability + metrics` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-WEB-AOC-19-001 | Export metrics (`ingestion_write_total`, `aoc_violation_total`, signature verification counters) and tracing spans matching Conseiller naming. Ensure structured logging includes tenant, source vendor, upstream id, and content hash. | +> Docs alignment (2025-10-26): Metrics/traces/log schema in `docs/observability/observability.md`. | EXCITITOR-WEB-AOC-19-003 `Guard + schema test harness` | TODO | QA Guild | EXCITITOR-WEB-AOC-19-001 | Add unit/integration tests for schema validation, forbidden field rejection (`ERR_AOC_001/006/007`), and supersedes behavior using CycloneDX-VEX & CSAF fixtures with deterministic expectations. | +> Docs alignment (2025-10-26): Error codes + CLI verification in `docs/cli/cli-reference.md`. | EXCITITOR-WEB-AOC-19-004 `Batch ingest validation` | TODO | Excititor WebService Guild, QA Guild | EXCITITOR-WEB-AOC-19-003, EXCITITOR-CORE-AOC-19-002 | Build large fixture ingest covering mixed VEX statuses, verifying raw storage parity, metrics, and CLI `aoc verify` compatibility. Document load test/runbook updates. | +> Docs alignment (2025-10-26): Offline/air-gap workflows captured in `docs/deploy/containers.md` §5. ## Policy Engine v2 @@ -25,8 +29,6 @@ | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| EXCITITOR-GRAPH-21-003 `Inspector fetch API` | TODO | Excititor WebService Guild, Cartographer Guild | EXCITITOR-GRAPH-21-001 | Expose batch endpoints for retrieving VEX statements + advisory metadata by PURL/component for Graph Explorer inspector. | -| EXCITITOR-GRAPH-21-004 `Overlay subscription` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-21-003, CARTO-GRAPH-21-007 | Emit events (NATS/Redis) when new VEX docs land so Cartographer can refresh overlays; include tenant/policy references. | ## Link-Not-Merge v1 @@ -43,11 +45,17 @@ | EXCITITOR-GRAPH-24-101 `VEX summary API` | TODO | Excititor WebService Guild | EXCITITOR-GRAPH-24-001 | Provide endpoints delivering VEX status summaries per component/asset for Vuln Explorer integration. | | EXCITITOR-GRAPH-24-102 `Evidence batch API` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-201 | Add batch VEX observation retrieval optimized for Graph overlays/tooltips. | +## VEX Lens (Sprint 30) + +| ID | Status | Owner(s) | Depends on | Notes | +|----|--------|----------|------------|-------| +| EXCITITOR-VEXLENS-30-001 `VEX evidence enrichers` | TODO | Excititor WebService Guild, VEX Lens Guild | EXCITITOR-VULN-29-001, VEXLENS-30-005 | Include issuer hints, signatures, and product trees in evidence payloads for VEX Lens; Label: VEX-Lens. | + ## Vulnerability Explorer (Sprint 29) | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| EXCITITOR-VULN-29-001 `VEX key normalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; backfill existing records. | +| EXCITITOR-VULN-29-001 `VEX key canonicalization` | TODO | Excititor WebService Guild | EXCITITOR-LNM-21-001 | Canonicalize (lossless) VEX advisory/product keys (map to `advisory_key`, capture product scopes); expose original sources in `links[]`; AOC-compliant: no merge, no derived fields, no suppression; backfill existing records. | | EXCITITOR-VULN-29-002 `Evidence retrieval` | TODO | Excititor WebService Guild | EXCITITOR-VULN-29-001, VULN-API-29-003 | Provide `/vuln/evidence/vex/{advisory_key}` returning raw VEX statements filtered by tenant/product scope for Explorer evidence tabs. | | EXCITITOR-VULN-29-004 `Observability` | TODO | Excititor WebService Guild, Observability Guild | EXCITITOR-VULN-29-001 | Add metrics/logs for VEX normalization, suppression scopes, withdrawn statements; emit events consumed by Vuln Explorer resolver. | diff --git a/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj b/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj index 82c79177..deba5310 100644 --- a/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj +++ b/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj @@ -7,7 +7,7 @@ true - + diff --git a/src/StellaOps.ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj b/src/StellaOps.ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj index 1ed273c2..e90142c1 100644 --- a/src/StellaOps.ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj +++ b/src/StellaOps.ExportCenter/StellaOps.ExportCenter.WebService/StellaOps.ExportCenter.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/StellaOps.ExportCenter.Worker.csproj b/src/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/StellaOps.ExportCenter.Worker.csproj index 5873d8ac..f79f1610 100644 --- a/src/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/StellaOps.ExportCenter.Worker.csproj +++ b/src/StellaOps.ExportCenter/StellaOps.ExportCenter.Worker/StellaOps.ExportCenter.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.Graph.Api/TASKS.md b/src/StellaOps.Graph.Api/TASKS.md index 0e7a3cc8..500b84df 100644 --- a/src/StellaOps.Graph.Api/TASKS.md +++ b/src/StellaOps.Graph.Api/TASKS.md @@ -6,9 +6,11 @@ | GRAPH-API-28-003 | TODO | Graph API Guild | GRAPH-API-28-001, GRAPH-INDEX-28-002..005 | Build query planner + cost estimator for `/graph/query`, stream tiles (nodes/edges/stats) progressively, enforce budgets, provide cursor tokens. | Query endpoint streams tiles deterministically, enforces budgets, surfaces truncation flags; integration tests cover large graphs. | | GRAPH-API-28-004 | TODO | Graph API Guild | GRAPH-API-28-003 | Implement `/graph/paths` with depth ≤6, constraint filters, heuristic shortest path search, and optional policy overlay rendering. | Paths API returns expected routes; policy overlay applied; guardrails enforced; tests cover over-budget errors. | | GRAPH-API-28-005 | TODO | Graph API Guild | GRAPH-INDEX-28-006, GRAPH-API-28-003 | Implement `/graph/diff` streaming added/removed/changed nodes/edges between SBOM snapshots; include overlay deltas and policy/VEX/advisory metadata. | Diff endpoint streams deterministic results; tests cover sample diffs; metrics record diff compute time. | -| GRAPH-API-28-006 | TODO | Graph API Guild | GRAPH-INDEX-28-002..005, POLICY-ENGINE-27-001 | Implement overlays (advisory, VEX, policy) with caching, partial materialization, and explain trace sampling for focused nodes. | Overlay pipeline delivers heatmap stats + explain samples; caches invalidate on policy/VEX/advisory change; tests cover concurrency. | +| GRAPH-API-28-006 | TODO | Graph API Guild | GRAPH-INDEX-28-002..005, POLICY-ENGINE-27-001 | Consume Policy Engine overlay contract (`POLICY-ENGINE-30-001..003`) and surface advisory/VEX/policy overlays with caching, partial materialization, and explain trace sampling for focused nodes. | Overlay pipeline delivers heatmap stats + explain samples; caches invalidate on policy/VEX/advisory change; tests cover concurrency. | | GRAPH-API-28-007 | TODO | Graph API Guild | GRAPH-API-28-003..006 | Implement exports (`graphml`, `csv`, `ndjson`, `png`, `svg`) with async job management, checksum manifests, and streaming downloads. | Export job API returns manifest + download URLs; tests validate formats; docs updated. | -| GRAPH-API-28-008 | TODO | Graph API Guild, Authority Guild | AUTH-GRAPH-26-001, AUTH-GRAPH-21-001 | Integrate RBAC scopes (`graph:read`, `graph:query`, `graph:export`), tenant headers, audit logging, and rate limiting. | Unauthorized access rejected; audit logs include query hash & scope; rate limits enforced; integration tests pass. | +| GRAPH-API-28-008 | TODO | Graph API Guild, Authority Guild | AUTH-GRAPH-26-001, AUTH-GRAPH-21-001 | Integrate RBAC scopes (`graph:read`, `graph:query`, `graph:export`), tenant headers, audit logging, and rate limiting. | Unauthorized access rejected; audit logs include query hash & scope; rate limits enforced; integration tests pass; scope checks use `StellaOpsScopes` constants (no string literals). | + +> 2025-10-26 — Waiting on Graph API host scaffolding. When endpoints land, ensure all scope enforcement relies on `StellaOpsScopes` before closing GRAPH-API-28-008. | GRAPH-API-28-009 | TODO | Graph API Guild, Observability Guild | GRAPH-API-28-002..007 | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. | Metrics exposed; dashboards live; alerts configured; docs updated. | | GRAPH-API-28-010 | TODO | Graph API Guild, QA Guild | GRAPH-API-28-002..007 | Build unit/integration/load tests with synthetic datasets (500k nodes/2M edges), fuzz query validation, verify determinism across runs. | Test suite green; load test report captured; determinism harness passes with fixed seed. | | GRAPH-API-28-011 | TODO | Graph API Guild, DevOps Guild | GRAPH-API-28-003..007 | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. | Deployment descriptors merged; gateway routes documented; offline kit instructions updated; smoke tests executed. | diff --git a/src/StellaOps.Graph.Indexer/TASKS.md b/src/StellaOps.Graph.Indexer/TASKS.md index 80f3804b..f44d3f40 100644 --- a/src/StellaOps.Graph.Indexer/TASKS.md +++ b/src/StellaOps.Graph.Indexer/TASKS.md @@ -3,7 +3,7 @@ |----|--------|----------|------------|-------------|---------------| | GRAPH-INDEX-28-001 | TODO | Graph Indexer Guild | SBOM-SERVICE-21-001, CARTO-GRAPH-21-001 | Define canonical node/edge schemas, attribute dictionaries, identity rules, and seed fixtures; publish schema doc. | Schema doc merged; identity property tests pass; fixtures committed for CI usage. | | GRAPH-INDEX-28-002 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, SBOM-SERVICE-21-002 | Implement SBOM ingest consumer producing artifact/package/file nodes and edges with `valid_from/valid_to`, scope metadata, and provenance links. | Ingest pipeline processes sample SBOMs deterministically; metrics recorded; unit tests cover identity stability. | -| GRAPH-INDEX-28-003 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, CONCELIER-CONSOLE-23-001 | Project Conseiller linksets into overlay tiles (`affected_by` edges, evidence refs) without mutating source observations; keep advisory aggregates in overlay store only. | Overlay documents generated deterministically; raw node/edge collections remain immutable; tests cover overlay refresh and eviction. | +| GRAPH-INDEX-28-003 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, CONCELIER-CONSOLE-23-001 | Project Concelier linksets into overlay tiles (`affected_by` edges, evidence refs) without mutating source observations; keep advisory aggregates in overlay store only. | Overlay documents generated deterministically; raw node/edge collections remain immutable; tests cover overlay refresh and eviction. | | GRAPH-INDEX-28-004 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-001, EXCITITOR-CONSOLE-23-001 | Integrate VEX statements (`vex_exempts` edges) with justification metadata and precedence markers for overlays. | VEX edges generated; conflicts resolved deterministically; tests cover status transitions. | | GRAPH-INDEX-28-005 | TODO | Graph Indexer Guild, Policy Guild | POLICY-ENGINE-27-001, POLICY-ENGINE-27-002 | Hydrate policy overlays into graph (`governs_with` nodes/edges) referencing effective findings and explain hashes for sampled nodes. | Overlay nodes stored with policy version id, severity, status; explain references captured; validation tests pass. | | GRAPH-INDEX-28-006 | TODO | Graph Indexer Guild | GRAPH-INDEX-28-002..005 | Generate graph snapshots per SBOM with lineage (`derived_from`), adjacency manifests, and metadata for diff jobs. | Snapshot documents produced; lineage recorded; tests assert diff readiness; metrics emitted. | diff --git a/src/StellaOps.Notifier/StellaOps.Notifier.WebService/StellaOps.Notifier.WebService.csproj b/src/StellaOps.Notifier/StellaOps.Notifier.WebService/StellaOps.Notifier.WebService.csproj index 65b2b077..90d30c8e 100644 --- a/src/StellaOps.Notifier/StellaOps.Notifier.WebService/StellaOps.Notifier.WebService.csproj +++ b/src/StellaOps.Notifier/StellaOps.Notifier.WebService/StellaOps.Notifier.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj b/src/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj index 227ecefc..0d61db52 100644 --- a/src/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj +++ b/src/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.Notify.Queue.Tests/StellaOps.Notify.Queue.Tests.csproj b/src/StellaOps.Notify.Queue.Tests/StellaOps.Notify.Queue.Tests.csproj index d4573c30..f4276e38 100644 --- a/src/StellaOps.Notify.Queue.Tests/StellaOps.Notify.Queue.Tests.csproj +++ b/src/StellaOps.Notify.Queue.Tests/StellaOps.Notify.Queue.Tests.csproj @@ -7,7 +7,7 @@ false - + diff --git a/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj b/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj index 7c4b247b..c57f16cb 100644 --- a/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj +++ b/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj @@ -5,14 +5,14 @@ enable - - - - - - - - + + + + + + + + diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj b/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj index ccf0cbe4..07da5734 100644 --- a/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj @@ -1,28 +1,28 @@ - - - net10.0 - enable - enable - false - - - - - - - - - - - - - - - - - - - Always - - - + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + Always + + + diff --git a/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj b/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj index c6563738..518d7842 100644 --- a/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj +++ b/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj @@ -1,18 +1,18 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Notify.Worker.Tests/StellaOps.Notify.Worker.Tests.csproj b/src/StellaOps.Notify.Worker.Tests/StellaOps.Notify.Worker.Tests.csproj index 785775c2..cb8ed36c 100644 --- a/src/StellaOps.Notify.Worker.Tests/StellaOps.Notify.Worker.Tests.csproj +++ b/src/StellaOps.Notify.Worker.Tests/StellaOps.Notify.Worker.Tests.csproj @@ -16,7 +16,7 @@ all - + diff --git a/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj b/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj index 0ee4ae2d..2d8013b3 100644 --- a/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj +++ b/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj @@ -6,11 +6,11 @@ Exe - - - - - + + + + + diff --git a/src/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/StellaOps.Orchestrator.WebService.csproj b/src/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/StellaOps.Orchestrator.WebService.csproj index 5f4779ba..1588ca57 100644 --- a/src/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/StellaOps.Orchestrator.WebService.csproj +++ b/src/StellaOps.Orchestrator/StellaOps.Orchestrator.WebService/StellaOps.Orchestrator.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.Orchestrator/StellaOps.Orchestrator.Worker/StellaOps.Orchestrator.Worker.csproj b/src/StellaOps.Orchestrator/StellaOps.Orchestrator.Worker/StellaOps.Orchestrator.Worker.csproj index 057adeb1..f2601d9f 100644 --- a/src/StellaOps.Orchestrator/StellaOps.Orchestrator.Worker/StellaOps.Orchestrator.Worker.csproj +++ b/src/StellaOps.Orchestrator/StellaOps.Orchestrator.Worker/StellaOps.Orchestrator.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.WebService/StellaOps.PacksRegistry.WebService.csproj b/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.WebService/StellaOps.PacksRegistry.WebService.csproj index 8721a7d3..71e052c6 100644 --- a/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.WebService/StellaOps.PacksRegistry.WebService.csproj +++ b/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.WebService/StellaOps.PacksRegistry.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.Worker/StellaOps.PacksRegistry.Worker.csproj b/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.Worker/StellaOps.PacksRegistry.Worker.csproj index 4eec11a4..ab1c918a 100644 --- a/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.Worker/StellaOps.PacksRegistry.Worker.csproj +++ b/src/StellaOps.PacksRegistry/StellaOps.PacksRegistry.Worker/StellaOps.PacksRegistry.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj b/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj index 5d477369..0b43c86f 100644 --- a/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj +++ b/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj @@ -1,21 +1,21 @@ - - - net10.0 - enable - enable - false - - - - - + + + net10.0 + enable + enable + false + + + + + - - + + - - - + + + diff --git a/src/StellaOps.Plugin/StellaOps.Plugin.csproj b/src/StellaOps.Plugin/StellaOps.Plugin.csproj index cee2eaba..9471a911 100644 --- a/src/StellaOps.Plugin/StellaOps.Plugin.csproj +++ b/src/StellaOps.Plugin/StellaOps.Plugin.csproj @@ -1,19 +1,19 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - + + + + net10.0 + enable + enable + + + + + + + + + + + + \ No newline at end of file diff --git a/src/StellaOps.Policy.Engine.Tests/PolicyCompilerTests.cs b/src/StellaOps.Policy.Engine.Tests/PolicyCompilerTests.cs new file mode 100644 index 00000000..b2a1fb44 --- /dev/null +++ b/src/StellaOps.Policy.Engine.Tests/PolicyCompilerTests.cs @@ -0,0 +1,104 @@ +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Policy; +using StellaOps.Policy.Engine.Compilation; +using Xunit; +using Xunit.Sdk; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class PolicyCompilerTests +{ + [Fact] + public void Compile_BaselinePolicy_Succeeds() + { + const string source = """ + policy "Baseline Production Policy" syntax "stella-dsl@1" { + metadata { + description = "Block critical, escalate high, enforce VEX justifications." + tags = ["baseline","production"] + } + + profile severity { + map vendor_weight { + source "GHSA" => +0.5 + source "OSV" => +0.0 + } + env exposure_adjustments { + if env.exposure == "internet" then +0.5 + } + } + + rule block_critical priority 5 { + when severity.normalized >= "Critical" + then status := "blocked" + because "Critical severity must be remediated before deploy." + } + + rule escalate_high_internet { + when severity.normalized == "High" + and env.exposure == "internet" + then escalate to severity_band("Critical") + because "High severity on internet-exposed asset escalates to critical." + } + + rule require_vex_justification { + when vex.any(status in ["not_affected","fixed"]) + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + annotate winning_statement := vex.latest().statementId + because "Respect strong vendor VEX claims." + } + + rule alert_warn_eol_runtime priority 1 { + when severity.normalized <= "Medium" + and sbom.has_tag("runtime:eol") + then warn message "Runtime marked as EOL; upgrade recommended." + because "Deprecated runtime should be upgraded." + } + } + """; + + var compiler = new PolicyCompiler(); + var result = compiler.Compile(source); + + if (!result.Success) + { + throw new Xunit.Sdk.XunitException($"Compilation failed: {Describe(result.Diagnostics)}"); + } + Assert.False(string.IsNullOrWhiteSpace(result.Checksum)); + Assert.NotEmpty(result.CanonicalRepresentation); + Assert.All(result.Diagnostics, issue => Assert.NotEqual(PolicyIssueSeverity.Error, issue.Severity)); + + var document = Assert.IsType(result.Document); + Assert.Equal("Baseline Production Policy", document.Name); + Assert.Equal("stella-dsl@1", document.Syntax); + Assert.Equal(4, document.Rules.Length); + Assert.Single(document.Profiles); + var firstAction = Assert.IsType(document.Rules[0].ThenActions[0]); + Assert.Equal("status", firstAction.Target[0]); + } + + [Fact] + public void Compile_MissingBecause_ReportsDiagnostic() + { + const string source = """ + policy "Incomplete" syntax "stella-dsl@1" { + rule missing_because { + when true + then status := "suppressed" + } + } + """; + + var compiler = new PolicyCompiler(); + var result = compiler.Compile(source); + + Assert.False(result.Success); + PolicyIssue diagnostic = result.Diagnostics.First(issue => issue.Code == "POLICY-DSL-PARSE-006"); + Assert.Equal(PolicyIssueSeverity.Error, diagnostic.Severity); + } + + private static string Describe(ImmutableArray issues) => + string.Join(" | ", issues.Select(issue => $"{issue.Severity}:{issue.Code}:{issue.Message}")); +} diff --git a/src/StellaOps.Policy.Engine.Tests/PolicyEvaluatorTests.cs b/src/StellaOps.Policy.Engine.Tests/PolicyEvaluatorTests.cs new file mode 100644 index 00000000..3535e120 --- /dev/null +++ b/src/StellaOps.Policy.Engine.Tests/PolicyEvaluatorTests.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Policy.Engine.Compilation; +using StellaOps.Policy.Engine.Evaluation; +using StellaOps.Policy.Engine.Services; +using Xunit; +using Xunit.Sdk; + +namespace StellaOps.Policy.Engine.Tests; + +public sealed class PolicyEvaluatorTests +{ + private static readonly string BaselinePolicy = """ +policy "Baseline Production Policy" syntax "stella-dsl@1" { + metadata { + description = "Block critical, escalate high, enforce VEX justifications." + tags = ["baseline","production"] + } + + profile severity { + map vendor_weight { + source "GHSA" => +0.5 + source "OSV" => +0.0 + } + env exposure_adjustments { + if env.exposure == "internet" then +0.5 + } + } + + rule block_critical priority 5 { + when severity.normalized >= "Critical" + then status := "blocked" + because "Critical severity must be remediated before deploy." + } + + rule escalate_high_internet { + when severity.normalized == "High" + and env.exposure == "internet" + then escalate to severity_band("Critical") + because "High severity on internet-exposed asset escalates to critical." + } + + rule require_vex_justification { + when vex.any(status in ["not_affected","fixed"]) + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + annotate winning_statement := vex.latest().statementId + because "Respect strong vendor VEX claims." + } + + rule alert_warn_eol_runtime priority 1 { + when severity.normalized <= "Medium" + and sbom.has_tag("runtime:eol") + then warn message "Runtime marked as EOL; upgrade recommended." + because "Deprecated runtime should be upgraded." + } +} +"""; + + private readonly PolicyCompiler compiler = new(); + private readonly PolicyEvaluationService evaluationService = new(); + + [Fact] + public void Evaluate_BlockCriticalRuleMatches() + { + var document = CompileBaseline(); + var context = CreateContext(severity: "Critical", exposure: "internal"); + + var result = evaluationService.Evaluate(document, context); + + Assert.True(result.Matched); + Assert.Equal("block_critical", result.RuleName); + Assert.Equal("blocked", result.Status); + } + + [Fact] + public void Evaluate_EscalateAdjustsSeverity() + { + var document = CompileBaseline(); + var context = CreateContext(severity: "High", exposure: "internet"); + + var result = evaluationService.Evaluate(document, context); + + Assert.True(result.Matched); + Assert.Equal("escalate_high_internet", result.RuleName); + Assert.Equal("affected", result.Status); + Assert.Equal("Critical", result.Severity); + } + + [Fact] + public void Evaluate_VexOverrideSetsStatusAndAnnotation() + { + var document = CompileBaseline(); + var statements = ImmutableArray.Create( + new PolicyEvaluationVexStatement("not_affected", "component_not_present", "stmt-001")); + var context = CreateContext("Medium", "internal") with + { + Vex = new PolicyEvaluationVexEvidence(statements) + }; + + var result = evaluationService.Evaluate(document, context); + + Assert.True(result.Matched); + Assert.Equal("require_vex_justification", result.RuleName); + Assert.Equal("not_affected", result.Status); + Assert.Equal("stmt-001", result.Annotations["winning_statement"]); + } + + [Fact] + public void Evaluate_WarnRuleEmitsWarning() + { + var document = CompileBaseline(); + var tags = ImmutableHashSet.Create("runtime:eol"); + var context = CreateContext("Medium", "internal") with + { + Sbom = new PolicyEvaluationSbom(tags) + }; + + var result = evaluationService.Evaluate(document, context); + + Assert.True(result.Matched); + Assert.Equal("alert_warn_eol_runtime", result.RuleName); + Assert.Equal("warned", result.Status); + Assert.Contains(result.Warnings, message => message.Contains("EOL", StringComparison.OrdinalIgnoreCase)); + } + + private PolicyIrDocument CompileBaseline() + { + var compilation = compiler.Compile(BaselinePolicy); + Assert.True(compilation.Success, Describe(compilation.Diagnostics)); + return Assert.IsType(compilation.Document); + } + + private static PolicyEvaluationContext CreateContext(string severity, string exposure) + { + return new PolicyEvaluationContext( + new PolicyEvaluationSeverity(severity), + new PolicyEvaluationEnvironment(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["exposure"] = exposure + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase)), + new PolicyEvaluationAdvisory("GHSA", ImmutableDictionary.Empty), + PolicyEvaluationVexEvidence.Empty, + new PolicyEvaluationSbom(ImmutableHashSet.Empty)); + } + + private static string Describe(ImmutableArray issues) => + string.Join(" | ", issues.Select(issue => $"{issue.Severity}:{issue.Code}:{issue.Message}")); +} diff --git a/src/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj b/src/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj new file mode 100644 index 00000000..4180c80e --- /dev/null +++ b/src/StellaOps.Policy.Engine.Tests/StellaOps.Policy.Engine.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + preview + enable + enable + true + + + + + + diff --git a/src/StellaOps.Policy.Engine/Compilation/DslToken.cs b/src/StellaOps.Policy.Engine/Compilation/DslToken.cs new file mode 100644 index 00000000..bcd107c7 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/DslToken.cs @@ -0,0 +1,160 @@ +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Policy.Engine.Compilation; + +/// +/// Represents a precise source location within a policy DSL document. +/// +public readonly struct SourceLocation : IEquatable, IComparable +{ + public SourceLocation(int offset, int line, int column) + { + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + if (line < 1) + { + throw new ArgumentOutOfRangeException(nameof(line)); + } + + if (column < 1) + { + throw new ArgumentOutOfRangeException(nameof(column)); + } + + Offset = offset; + Line = line; + Column = column; + } + + public int Offset { get; } + + public int Line { get; } + + public int Column { get; } + + public override string ToString() => $"(L{Line}, C{Column})"; + + public bool Equals(SourceLocation other) => + Offset == other.Offset && Line == other.Line && Column == other.Column; + + public override bool Equals([NotNullWhen(true)] object? obj) => + obj is SourceLocation other && Equals(other); + + public override int GetHashCode() => HashCode.Combine(Offset, Line, Column); + + public int CompareTo(SourceLocation other) => Offset.CompareTo(other.Offset); + + public static bool operator ==(SourceLocation left, SourceLocation right) => left.Equals(right); + + public static bool operator !=(SourceLocation left, SourceLocation right) => !left.Equals(right); + + public static bool operator <(SourceLocation left, SourceLocation right) => left.CompareTo(right) < 0; + + public static bool operator <=(SourceLocation left, SourceLocation right) => left.CompareTo(right) <= 0; + + public static bool operator >(SourceLocation left, SourceLocation right) => left.CompareTo(right) > 0; + + public static bool operator >=(SourceLocation left, SourceLocation right) => left.CompareTo(right) >= 0; +} + +/// +/// Represents a start/end location pair within a policy DSL source document. +/// +public readonly struct SourceSpan : IEquatable +{ + public SourceSpan(SourceLocation start, SourceLocation end) + { + if (start.Offset > end.Offset) + { + throw new ArgumentException("Start must not be after end.", nameof(start)); + } + + Start = start; + End = end; + } + + public SourceLocation Start { get; } + + public SourceLocation End { get; } + + public override string ToString() => $"{Start}->{End}"; + + public bool Equals(SourceSpan other) => Start.Equals(other.Start) && End.Equals(other.End); + + public override bool Equals([NotNullWhen(true)] object? obj) => obj is SourceSpan other && Equals(other); + + public override int GetHashCode() => HashCode.Combine(Start, End); + + public static SourceSpan Combine(SourceSpan first, SourceSpan second) + { + var start = first.Start <= second.Start ? first.Start : second.Start; + var end = first.End >= second.End ? first.End : second.End; + return new SourceSpan(start, end); + } +} + +internal enum TokenKind +{ + EndOfFile = 0, + Identifier, + StringLiteral, + NumberLiteral, + BooleanLiteral, + LeftBrace, + RightBrace, + LeftParen, + RightParen, + LeftBracket, + RightBracket, + Comma, + Semicolon, + Colon, + Arrow, // => + Assign, // = + Define, // := + Dot, + KeywordPolicy, + KeywordSyntax, + KeywordMetadata, + KeywordProfile, + KeywordRule, + KeywordMap, + KeywordSource, + KeywordEnv, + KeywordIf, + KeywordThen, + KeywordWhen, + KeywordAnd, + KeywordOr, + KeywordNot, + KeywordPriority, + KeywordElse, + KeywordBecause, + KeywordSettings, + KeywordIgnore, + KeywordUntil, + KeywordEscalate, + KeywordTo, + KeywordRequireVex, + KeywordWarn, + KeywordMessage, + KeywordDefer, + KeywordAnnotate, + KeywordIn, + EqualEqual, + NotEqual, + LessThan, + LessThanOrEqual, + GreaterThan, + GreaterThanOrEqual, + Unknown, +} + +internal readonly record struct DslToken( + TokenKind Kind, + string Text, + SourceSpan Span, + object? Value = null); diff --git a/src/StellaOps.Policy.Engine/Compilation/DslTokenizer.cs b/src/StellaOps.Policy.Engine/Compilation/DslTokenizer.cs new file mode 100644 index 00000000..01f68ce8 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/DslTokenizer.cs @@ -0,0 +1,576 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using StellaOps.Policy; + +namespace StellaOps.Policy.Engine.Compilation; + +internal static class DslTokenizer +{ + public static TokenizerResult Tokenize(string source) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + var tokens = ImmutableArray.CreateBuilder(); + var diagnostics = ImmutableArray.CreateBuilder(); + + var index = 0; + var line = 1; + var column = 1; + + while (index < source.Length) + { + var current = source[index]; + if (char.IsWhiteSpace(current)) + { + (index, line, column) = AdvanceWhitespace(source, index, line, column); + continue; + } + + if (current == '/' && index + 1 < source.Length) + { + if (source[index + 1] == '/') + { + (index, line, column) = SkipSingleLineComment(source, index + 2, line, column + 2); + continue; + } + + if (source[index + 1] == '*') + { + (index, line, column) = SkipMultiLineComment(source, index + 2, line, column + 2, diagnostics); + continue; + } + } + + var startLocation = new SourceLocation(index, line, column); + switch (current) + { + case '{': + tokens.Add(CreateToken(TokenKind.LeftBrace, "{", startLocation, ref index, ref column)); + break; + case '}': + tokens.Add(CreateToken(TokenKind.RightBrace, "}", startLocation, ref index, ref column)); + break; + case '(': + tokens.Add(CreateToken(TokenKind.LeftParen, "(", startLocation, ref index, ref column)); + break; + case ')': + tokens.Add(CreateToken(TokenKind.RightParen, ")", startLocation, ref index, ref column)); + break; + case '[': + tokens.Add(CreateToken(TokenKind.LeftBracket, "[", startLocation, ref index, ref column)); + break; + case ']': + tokens.Add(CreateToken(TokenKind.RightBracket, "]", startLocation, ref index, ref column)); + break; + case ',': + tokens.Add(CreateToken(TokenKind.Comma, ",", startLocation, ref index, ref column)); + break; + case ';': + tokens.Add(CreateToken(TokenKind.Semicolon, ";", startLocation, ref index, ref column)); + break; + case ':': + { + if (Match(source, index + 1, '=')) + { + tokens.Add(CreateToken(TokenKind.Define, ":=", startLocation, ref index, ref column, advance: 2)); + } + else + { + tokens.Add(CreateToken(TokenKind.Colon, ":", startLocation, ref index, ref column)); + } + + break; + } + case '=': + { + if (Match(source, index + 1, '>')) + { + tokens.Add(CreateToken(TokenKind.Arrow, "=>", startLocation, ref index, ref column, advance: 2)); + } + else if (Match(source, index + 1, '=')) + { + tokens.Add(CreateToken(TokenKind.EqualEqual, "==", startLocation, ref index, ref column, advance: 2)); + } + else + { + tokens.Add(CreateToken(TokenKind.Assign, "=", startLocation, ref index, ref column)); + } + + break; + } + case '!': + { + if (Match(source, index + 1, '=')) + { + tokens.Add(CreateToken(TokenKind.NotEqual, "!=", startLocation, ref index, ref column, advance: 2)); + } + else + { + ReportUnexpectedCharacter(diagnostics, current, startLocation); + index++; + column++; + } + + break; + } + case '<': + { + if (Match(source, index + 1, '=')) + { + tokens.Add(CreateToken(TokenKind.LessThanOrEqual, "<=", startLocation, ref index, ref column, advance: 2)); + } + else + { + tokens.Add(CreateToken(TokenKind.LessThan, "<", startLocation, ref index, ref column)); + } + + break; + } + case '>': + { + if (Match(source, index + 1, '=')) + { + tokens.Add(CreateToken(TokenKind.GreaterThanOrEqual, ">=", startLocation, ref index, ref column, advance: 2)); + } + else + { + tokens.Add(CreateToken(TokenKind.GreaterThan, ">", startLocation, ref index, ref column)); + } + + break; + } + case '.': + tokens.Add(CreateToken(TokenKind.Dot, ".", startLocation, ref index, ref column)); + break; + case '"': + TokenizeString(source, ref index, ref line, ref column, startLocation, tokens, diagnostics); + break; + case '+': + case '-': + { + if (index + 1 < source.Length && char.IsDigit(source[index + 1])) + { + TokenizeNumber(source, ref index, ref line, ref column, startLocation, tokens, diagnostics); + } + else + { + ReportUnexpectedCharacter(diagnostics, current, startLocation); + index++; + column++; + } + + break; + } + default: + { + if (char.IsDigit(current)) + { + TokenizeNumber(source, ref index, ref line, ref column, startLocation, tokens, diagnostics); + } + else if (IsIdentifierStart(current)) + { + TokenizeIdentifierOrKeyword(source, ref index, ref line, ref column, startLocation, tokens); + } + else + { + ReportUnexpectedCharacter(diagnostics, current, startLocation); + index++; + column++; + } + + break; + } + } + } + + var eofLocation = new SourceLocation(index, line, column); + tokens.Add(new DslToken(TokenKind.EndOfFile, string.Empty, new SourceSpan(eofLocation, eofLocation))); + + return new TokenizerResult(tokens.ToImmutable(), diagnostics.ToImmutable()); + } + + private static void TokenizeString( + string source, + ref int index, + ref int line, + ref int column, + SourceLocation start, + ImmutableArray.Builder tokens, + ImmutableArray.Builder diagnostics) + { + var builder = new StringBuilder(); + var i = index + 1; + var currentLine = line; + var currentColumn = column + 1; + + while (i < source.Length) + { + var ch = source[i]; + if (ch == '"') + { + var end = new SourceLocation(i + 1, currentLine, currentColumn + 1); + index = i + 1; + column = currentColumn + 1; + tokens.Add(new DslToken(TokenKind.StringLiteral, builder.ToString(), new SourceSpan(start, end), builder.ToString())); + return; + } + + if (ch == '\\') + { + if (i + 1 >= source.Length) + { + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnterminatedString, "Unterminated string literal.", $"@{start.Line}:{start.Column}")); + index = source.Length; + line = currentLine; + column = currentColumn; + return; + } + + var escape = source[i + 1]; + switch (escape) + { + case '\\': + builder.Append('\\'); + break; + case '"': + builder.Append('"'); + break; + case 'n': + builder.Append('\n'); + break; + case 'r': + builder.Append('\r'); + break; + case 't': + builder.Append('\t'); + break; + default: + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.InvalidEscapeSequence, $"Invalid escape sequence '\\{escape}'.", $"@{currentLine}:{currentColumn}")); + builder.Append(escape); + break; + } + + i += 2; + currentColumn += 2; + continue; + } + + if (ch == '\r' || ch == '\n') + { + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnterminatedString, "Unterminated string literal.", $"@{start.Line}:{start.Column}")); + (index, line, column) = AdvanceWhitespace(source, i, currentLine, currentColumn); + return; + } + + builder.Append(ch); + i++; + currentColumn++; + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnterminatedString, "Unterminated string literal.", $"@{start.Line}:{start.Column}")); + index = source.Length; + line = currentLine; + column = currentColumn; + } + + private static void TokenizeNumber( + string source, + ref int index, + ref int line, + ref int column, + SourceLocation start, + ImmutableArray.Builder tokens, + ImmutableArray.Builder diagnostics) + { + var i = index; + var hasDecimal = false; + + if (source[i] == '+' || source[i] == '-') + { + i++; + } + + while (i < source.Length) + { + var ch = source[i]; + if (char.IsDigit(ch)) + { + i++; + continue; + } + + if (ch == '.') + { + if (hasDecimal) + { + break; + } + + hasDecimal = true; + i++; + continue; + } + + break; + } + + var percent = false; + if (i < source.Length && source[i] == '%') + { + percent = true; + i++; + } + + var text = source.Substring(index, i - index); + if (!decimal.TryParse(text, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var value)) + { + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.InvalidNumber, $"Invalid numeric literal '{text}'.", $"@{start.Line}:{start.Column}")); + index = i; + column += i - index; + return; + } + + if (percent) + { + value /= 100m; + } + + var end = new SourceLocation(i, line, column + (i - index)); + tokens.Add(new DslToken(TokenKind.NumberLiteral, text, new SourceSpan(start, end), value)); + column += i - index; + index = i; + } + + private static void TokenizeIdentifierOrKeyword( + string source, + ref int index, + ref int line, + ref int column, + SourceLocation start, + ImmutableArray.Builder tokens) + { + var i = index + 1; + while (i < source.Length && IsIdentifierPart(source[i])) + { + i++; + } + + var text = source.Substring(index, i - index); + var kind = GetKeywordKind(text); + + if (kind == TokenKind.BooleanLiteral) + { + var value = string.Equals(text, "true", StringComparison.Ordinal); + var end = new SourceLocation(i, line, column + (i - index)); + tokens.Add(new DslToken(TokenKind.BooleanLiteral, text, new SourceSpan(start, end), value)); + } + else if (kind == TokenKind.Identifier) + { + var end = new SourceLocation(i, line, column + (i - index)); + tokens.Add(new DslToken(TokenKind.Identifier, text, new SourceSpan(start, end))); + } + else + { + var end = new SourceLocation(i, line, column + (i - index)); + tokens.Add(new DslToken(kind, text, new SourceSpan(start, end))); + } + + column += i - index; + index = i; + } + + private static TokenKind GetKeywordKind(string text) + { + return text switch + { + "policy" => TokenKind.KeywordPolicy, + "syntax" => TokenKind.KeywordSyntax, + "metadata" => TokenKind.KeywordMetadata, + "profile" => TokenKind.KeywordProfile, + "rule" => TokenKind.KeywordRule, + "map" => TokenKind.KeywordMap, + "source" => TokenKind.KeywordSource, + "env" => TokenKind.Identifier, + "if" => TokenKind.KeywordIf, + "then" => TokenKind.KeywordThen, + "when" => TokenKind.KeywordWhen, + "and" => TokenKind.KeywordAnd, + "or" => TokenKind.KeywordOr, + "not" => TokenKind.KeywordNot, + "priority" => TokenKind.KeywordPriority, + "else" => TokenKind.KeywordElse, + "because" => TokenKind.KeywordBecause, + "settings" => TokenKind.KeywordSettings, + "ignore" => TokenKind.KeywordIgnore, + "until" => TokenKind.KeywordUntil, + "escalate" => TokenKind.KeywordEscalate, + "to" => TokenKind.KeywordTo, + "requireVex" => TokenKind.KeywordRequireVex, + "warn" => TokenKind.KeywordWarn, + "message" => TokenKind.KeywordMessage, + "defer" => TokenKind.KeywordDefer, + "annotate" => TokenKind.KeywordAnnotate, + "in" => TokenKind.KeywordIn, + "true" => TokenKind.BooleanLiteral, + "false" => TokenKind.BooleanLiteral, + _ => TokenKind.Identifier, + }; + } + + private static bool IsIdentifierStart(char ch) => char.IsLetter(ch) || ch == '_'; + + private static bool IsIdentifierPart(char ch) => char.IsLetterOrDigit(ch) || ch == '_' || ch == '-'; + + private static (int Index, int Line, int Column) AdvanceWhitespace(string source, int index, int line, int column) + { + var i = index; + var currentLine = line; + var currentColumn = column; + + while (i < source.Length) + { + var ch = source[i]; + if (ch == '\r') + { + if (i + 1 < source.Length && source[i + 1] == '\n') + { + i += 2; + } + else + { + i++; + } + + currentLine++; + currentColumn = 1; + continue; + } + + if (ch == '\n') + { + i++; + currentLine++; + currentColumn = 1; + continue; + } + + if (!char.IsWhiteSpace(ch)) + { + break; + } + + i++; + currentColumn++; + } + + return (i, currentLine, currentColumn); + } + + private static (int Index, int Line, int Column) SkipSingleLineComment(string source, int index, int line, int column) + { + var i = index; + var currentLine = line; + var currentColumn = column; + + while (i < source.Length) + { + var ch = source[i]; + if (ch == '\r' || ch == '\n') + { + return AdvanceWhitespace(source, i, currentLine, currentColumn); + } + + i++; + currentColumn++; + } + + return (i, currentLine, currentColumn); + } + + private static (int Index, int Line, int Column) SkipMultiLineComment( + string source, + int index, + int line, + int column, + ImmutableArray.Builder diagnostics) + { + var i = index; + var currentLine = line; + var currentColumn = column; + + while (i < source.Length) + { + var ch = source[i]; + if (ch == '*' && i + 1 < source.Length && source[i + 1] == '/') + { + return (i + 2, currentLine, currentColumn + 2); + } + + if (ch == '\r') + { + if (i + 1 < source.Length && source[i + 1] == '\n') + { + i += 2; + } + else + { + i++; + } + + currentLine++; + currentColumn = 1; + continue; + } + + if (ch == '\n') + { + i++; + currentLine++; + currentColumn = 1; + continue; + } + + i++; + currentColumn++; + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnexpectedCharacter, "Unterminated comment block.", $"@{line}:{column}")); + return (source.Length, currentLine, currentColumn); + } + + private static DslToken CreateToken( + TokenKind kind, + string text, + SourceLocation start, + ref int index, + ref int column, + int advance = 1) + { + var end = new SourceLocation(index + advance, start.Line, start.Column + advance); + index += advance; + column += advance; + return new DslToken(kind, text, new SourceSpan(start, end)); + } + + private static void ReportUnexpectedCharacter( + ImmutableArray.Builder diagnostics, + char ch, + SourceLocation location) + { + diagnostics.Add(PolicyIssue.Error( + PolicyDslDiagnosticCodes.UnexpectedCharacter, + $"Unexpected character '{ch}'.", + $"@{location.Line}:{location.Column}")); + } + + private static bool Match(string source, int index, char expected) => + index < source.Length && source[index] == expected; +} + +internal readonly record struct TokenizerResult( + ImmutableArray Tokens, + ImmutableArray Diagnostics); diff --git a/src/StellaOps.Policy.Engine/Compilation/PolicyCompiler.cs b/src/StellaOps.Policy.Engine/Compilation/PolicyCompiler.cs new file mode 100644 index 00000000..48ead072 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/PolicyCompiler.cs @@ -0,0 +1,169 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using StellaOps.Policy; + +namespace StellaOps.Policy.Engine.Compilation; + +public sealed class PolicyCompiler +{ + public PolicyCompilationResult Compile(string source) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + var parseResult = PolicyParser.Parse(source); + if (parseResult.Document is null) + { + return new PolicyCompilationResult( + Success: false, + Document: null, + Checksum: null, + CanonicalRepresentation: ImmutableArray.Empty, + Diagnostics: parseResult.Diagnostics); + } + + if (parseResult.Diagnostics.Any(static issue => issue.Severity == PolicyIssueSeverity.Error)) + { + return new PolicyCompilationResult( + Success: false, + Document: null, + Checksum: null, + CanonicalRepresentation: ImmutableArray.Empty, + Diagnostics: parseResult.Diagnostics); + } + + var irDocument = BuildIntermediateRepresentation(parseResult.Document); + var canonical = PolicyIrSerializer.Serialize(irDocument); + var checksum = Convert.ToHexString(SHA256.HashData(canonical.AsSpan())).ToLowerInvariant(); + + return new PolicyCompilationResult( + Success: true, + Document: irDocument, + Checksum: checksum, + CanonicalRepresentation: canonical, + Diagnostics: parseResult.Diagnostics); + } + + private static PolicyIrDocument BuildIntermediateRepresentation(PolicyDocumentNode node) + { + var metadata = node.Metadata + .OrderBy(static kvp => kvp.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static kvp => kvp.Key, kvp => ToIrLiteral(kvp.Value), StringComparer.Ordinal); + + var settings = node.Settings + .OrderBy(static kvp => kvp.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static kvp => kvp.Key, kvp => ToIrLiteral(kvp.Value), StringComparer.Ordinal); + + var profiles = ImmutableArray.CreateBuilder(node.Profiles.Length); + foreach (var profile in node.Profiles) + { + var maps = ImmutableArray.CreateBuilder(); + var envs = ImmutableArray.CreateBuilder(); + var scalars = ImmutableArray.CreateBuilder(); + + foreach (var item in profile.Items) + { + switch (item) + { + case PolicyProfileMapNode map: + maps.Add(new PolicyIrProfileMap( + map.Name, + map.Entries + .Select(entry => new PolicyIrProfileMapEntry(entry.Source, entry.Weight)) + .ToImmutableArray())); + break; + case PolicyProfileEnvNode env: + envs.Add(new PolicyIrProfileEnv( + env.Name, + env.Entries + .Select(entry => new PolicyIrProfileEnvEntry(entry.Condition, entry.Weight)) + .ToImmutableArray())); + break; + case PolicyProfileScalarNode scalar: + scalars.Add(new PolicyIrProfileScalar(scalar.Name, ToIrLiteral(scalar.Value))); + break; + } + } + + profiles.Add(new PolicyIrProfile( + profile.Name, + maps.ToImmutable(), + envs.ToImmutable(), + scalars.ToImmutable())); + } + + var rules = ImmutableArray.CreateBuilder(node.Rules.Length); + foreach (var rule in node.Rules) + { + var thenActions = ImmutableArray.CreateBuilder(rule.ThenActions.Length); + foreach (var action in rule.ThenActions) + { + var converted = ToIrAction(action); + if (converted is not null) + { + thenActions.Add(converted); + } + } + + var elseActions = ImmutableArray.CreateBuilder(rule.ElseActions.Length); + foreach (var action in rule.ElseActions) + { + var converted = ToIrAction(action); + if (converted is not null) + { + elseActions.Add(converted); + } + } + + rules.Add(new PolicyIrRule( + rule.Name, + rule.Priority, + rule.When, + thenActions.ToImmutable(), + elseActions.ToImmutable(), + rule.Because ?? string.Empty)); + } + + return new PolicyIrDocument( + node.Name, + node.Syntax, + metadata, + profiles.ToImmutable(), + settings, + rules.ToImmutable()); + } + + private static PolicyIrLiteral ToIrLiteral(PolicyLiteralValue value) => value switch + { + PolicyStringLiteral s => new PolicyIrStringLiteral(s.Value), + PolicyNumberLiteral n => new PolicyIrNumberLiteral(n.Value), + PolicyBooleanLiteral b => new PolicyIrBooleanLiteral(b.Value), + PolicyListLiteral list => new PolicyIrListLiteral(list.Items.Select(ToIrLiteral).ToImmutableArray()), + _ => new PolicyIrStringLiteral(string.Empty), + }; + + private static PolicyIrAction? ToIrAction(PolicyActionNode action) => action switch + { + PolicyAssignmentActionNode assign => new PolicyIrAssignmentAction(assign.Target.Segments, assign.Value), + PolicyAnnotateActionNode annotate => new PolicyIrAnnotateAction(annotate.Target.Segments, annotate.Value), + PolicyIgnoreActionNode ignore => new PolicyIrIgnoreAction(ignore.Until, ignore.Because), + PolicyEscalateActionNode escalate => new PolicyIrEscalateAction(escalate.To, escalate.When), + PolicyRequireVexActionNode require => new PolicyIrRequireVexAction( + require.Conditions + .OrderBy(static kvp => kvp.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal)), + PolicyWarnActionNode warn => new PolicyIrWarnAction(warn.Message), + PolicyDeferActionNode defer => new PolicyIrDeferAction(defer.Until), + _ => null, + }; +} + +public sealed record PolicyCompilationResult( + bool Success, + PolicyIrDocument? Document, + string? Checksum, + ImmutableArray CanonicalRepresentation, + ImmutableArray Diagnostics); diff --git a/src/StellaOps.Policy.Engine/Compilation/PolicyDslDiagnosticCodes.cs b/src/StellaOps.Policy.Engine/Compilation/PolicyDslDiagnosticCodes.cs new file mode 100644 index 00000000..242e8528 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/PolicyDslDiagnosticCodes.cs @@ -0,0 +1,19 @@ +namespace StellaOps.Policy.Engine.Compilation; + +internal static class PolicyDslDiagnosticCodes +{ + public const string UnexpectedCharacter = "POLICY-DSL-LEX-001"; + public const string UnterminatedString = "POLICY-DSL-LEX-002"; + public const string InvalidEscapeSequence = "POLICY-DSL-LEX-003"; + public const string InvalidNumber = "POLICY-DSL-LEX-004"; + public const string UnexpectedToken = "POLICY-DSL-PARSE-001"; + public const string DuplicateSection = "POLICY-DSL-PARSE-002"; + public const string MissingPolicyHeader = "POLICY-DSL-PARSE-003"; + public const string UnsupportedSyntaxVersion = "POLICY-DSL-PARSE-004"; + public const string DuplicateRuleName = "POLICY-DSL-PARSE-005"; + public const string MissingBecauseClause = "POLICY-DSL-PARSE-006"; + public const string MissingTerminator = "POLICY-DSL-PARSE-007"; + public const string InvalidAction = "POLICY-DSL-PARSE-008"; + public const string InvalidLiteral = "POLICY-DSL-PARSE-009"; + public const string UnexpectedSection = "POLICY-DSL-PARSE-010"; +} diff --git a/src/StellaOps.Policy.Engine/Compilation/PolicyIr.cs b/src/StellaOps.Policy.Engine/Compilation/PolicyIr.cs new file mode 100644 index 00000000..6f9aec83 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/PolicyIr.cs @@ -0,0 +1,61 @@ +using System.Collections.Immutable; + +namespace StellaOps.Policy.Engine.Compilation; + +public sealed record PolicyIrDocument( + string Name, + string Syntax, + ImmutableSortedDictionary Metadata, + ImmutableArray Profiles, + ImmutableSortedDictionary Settings, + ImmutableArray Rules); + +public abstract record PolicyIrLiteral; + +public sealed record PolicyIrStringLiteral(string Value) : PolicyIrLiteral; + +public sealed record PolicyIrNumberLiteral(decimal Value) : PolicyIrLiteral; + +public sealed record PolicyIrBooleanLiteral(bool Value) : PolicyIrLiteral; + +public sealed record PolicyIrListLiteral(ImmutableArray Items) : PolicyIrLiteral; + +public sealed record PolicyIrProfile( + string Name, + ImmutableArray Maps, + ImmutableArray Environments, + ImmutableArray Scalars); + +public sealed record PolicyIrProfileMap(string Name, ImmutableArray Entries); + +public sealed record PolicyIrProfileMapEntry(string Source, decimal Weight); + +public sealed record PolicyIrProfileEnv(string Name, ImmutableArray Entries); + +public sealed record PolicyIrProfileEnvEntry(PolicyExpression Condition, decimal Weight); + +public sealed record PolicyIrProfileScalar(string Name, PolicyIrLiteral Value); + +public sealed record PolicyIrRule( + string Name, + int Priority, + PolicyExpression When, + ImmutableArray ThenActions, + ImmutableArray ElseActions, + string Because); + +public abstract record PolicyIrAction; + +public sealed record PolicyIrAssignmentAction(ImmutableArray Target, PolicyExpression Value) : PolicyIrAction; + +public sealed record PolicyIrAnnotateAction(ImmutableArray Target, PolicyExpression Value) : PolicyIrAction; + +public sealed record PolicyIrIgnoreAction(PolicyExpression? Until, string? Because) : PolicyIrAction; + +public sealed record PolicyIrEscalateAction(PolicyExpression? To, PolicyExpression? When) : PolicyIrAction; + +public sealed record PolicyIrRequireVexAction(ImmutableSortedDictionary Conditions) : PolicyIrAction; + +public sealed record PolicyIrWarnAction(PolicyExpression? Message) : PolicyIrAction; + +public sealed record PolicyIrDeferAction(PolicyExpression? Until) : PolicyIrAction; diff --git a/src/StellaOps.Policy.Engine/Compilation/PolicyIrSerializer.cs b/src/StellaOps.Policy.Engine/Compilation/PolicyIrSerializer.cs new file mode 100644 index 00000000..fc7b97be --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/PolicyIrSerializer.cs @@ -0,0 +1,415 @@ +using System.Buffers; +using System.Collections.Immutable; +using System.Text.Json; + +namespace StellaOps.Policy.Engine.Compilation; + +internal static class PolicyIrSerializer +{ + public static ImmutableArray Serialize(PolicyIrDocument document) + { + var buffer = new ArrayBufferWriter(); + using var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions + { + Indented = false, + SkipValidation = false + }); + + WriteDocument(writer, document); + writer.Flush(); + + return buffer.WrittenSpan.ToArray().ToImmutableArray(); + } + + private static void WriteDocument(Utf8JsonWriter writer, PolicyIrDocument document) + { + writer.WriteStartObject(); + writer.WriteString("name", document.Name); + writer.WriteString("syntax", document.Syntax); + + writer.WritePropertyName("metadata"); + WriteLiteralDictionary(writer, document.Metadata); + + writer.WritePropertyName("profiles"); + writer.WriteStartArray(); + foreach (var profile in document.Profiles) + { + WriteProfile(writer, profile); + } + + writer.WriteEndArray(); + + writer.WritePropertyName("settings"); + WriteLiteralDictionary(writer, document.Settings); + + writer.WritePropertyName("rules"); + writer.WriteStartArray(); + foreach (var rule in document.Rules) + { + WriteRule(writer, rule); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + private static void WriteProfile(Utf8JsonWriter writer, PolicyIrProfile profile) + { + writer.WriteStartObject(); + writer.WriteString("name", profile.Name); + + writer.WritePropertyName("maps"); + writer.WriteStartArray(); + foreach (var map in profile.Maps) + { + writer.WriteStartObject(); + writer.WriteString("name", map.Name); + writer.WritePropertyName("entries"); + writer.WriteStartArray(); + foreach (var entry in map.Entries) + { + writer.WriteStartObject(); + writer.WriteString("source", entry.Source); + writer.WriteNumber("weight", entry.Weight); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + writer.WritePropertyName("env"); + writer.WriteStartArray(); + foreach (var env in profile.Environments) + { + writer.WriteStartObject(); + writer.WriteString("name", env.Name); + writer.WritePropertyName("entries"); + writer.WriteStartArray(); + foreach (var entry in env.Entries) + { + writer.WriteStartObject(); + writer.WritePropertyName("condition"); + WriteExpression(writer, entry.Condition); + writer.WriteNumber("weight", entry.Weight); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + + writer.WritePropertyName("scalars"); + writer.WriteStartArray(); + foreach (var scalar in profile.Scalars) + { + writer.WriteStartObject(); + writer.WriteString("name", scalar.Name); + writer.WritePropertyName("value"); + WriteLiteral(writer, scalar.Value); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + private static void WriteRule(Utf8JsonWriter writer, PolicyIrRule rule) + { + writer.WriteStartObject(); + writer.WriteString("name", rule.Name); + writer.WriteNumber("priority", rule.Priority); + writer.WritePropertyName("when"); + WriteExpression(writer, rule.When); + + writer.WritePropertyName("then"); + WriteActions(writer, rule.ThenActions); + + writer.WritePropertyName("else"); + WriteActions(writer, rule.ElseActions); + + writer.WriteString("because", rule.Because); + writer.WriteEndObject(); + } + + private static void WriteActions(Utf8JsonWriter writer, ImmutableArray actions) + { + writer.WriteStartArray(); + foreach (var action in actions) + { + WriteAction(writer, action); + } + + writer.WriteEndArray(); + } + + private static void WriteAction(Utf8JsonWriter writer, PolicyIrAction action) + { + switch (action) + { + case PolicyIrAssignmentAction assign: + writer.WriteStartObject(); + writer.WriteString("type", "assign"); + WriteReference(writer, assign.Target); + writer.WritePropertyName("value"); + WriteExpression(writer, assign.Value); + writer.WriteEndObject(); + break; + case PolicyIrAnnotateAction annotate: + writer.WriteStartObject(); + writer.WriteString("type", "annotate"); + WriteReference(writer, annotate.Target); + writer.WritePropertyName("value"); + WriteExpression(writer, annotate.Value); + writer.WriteEndObject(); + break; + case PolicyIrIgnoreAction ignore: + writer.WriteStartObject(); + writer.WriteString("type", "ignore"); + writer.WritePropertyName("until"); + WriteOptionalExpression(writer, ignore.Until); + writer.WriteString("because", ignore.Because ?? string.Empty); + writer.WriteEndObject(); + break; + case PolicyIrEscalateAction escalate: + writer.WriteStartObject(); + writer.WriteString("type", "escalate"); + writer.WritePropertyName("to"); + WriteOptionalExpression(writer, escalate.To); + writer.WritePropertyName("when"); + WriteOptionalExpression(writer, escalate.When); + writer.WriteEndObject(); + break; + case PolicyIrRequireVexAction require: + writer.WriteStartObject(); + writer.WriteString("type", "requireVex"); + writer.WritePropertyName("conditions"); + writer.WriteStartObject(); + foreach (var kvp in require.Conditions) + { + writer.WritePropertyName(kvp.Key); + WriteExpression(writer, kvp.Value); + } + + writer.WriteEndObject(); + writer.WriteEndObject(); + break; + case PolicyIrWarnAction warn: + writer.WriteStartObject(); + writer.WriteString("type", "warn"); + writer.WritePropertyName("message"); + WriteOptionalExpression(writer, warn.Message); + writer.WriteEndObject(); + break; + case PolicyIrDeferAction defer: + writer.WriteStartObject(); + writer.WriteString("type", "defer"); + writer.WritePropertyName("until"); + WriteOptionalExpression(writer, defer.Until); + writer.WriteEndObject(); + break; + } + } + + private static void WriteReference(Utf8JsonWriter writer, ImmutableArray segments) + { + writer.WritePropertyName("target"); + writer.WriteStartArray(); + foreach (var segment in segments) + { + writer.WriteStringValue(segment); + } + + writer.WriteEndArray(); + } + + private static void WriteOptionalExpression(Utf8JsonWriter writer, PolicyExpression? expression) + { + if (expression is null) + { + writer.WriteNullValue(); + return; + } + + WriteExpression(writer, expression); + } + + private static void WriteExpression(Utf8JsonWriter writer, PolicyExpression expression) + { + switch (expression) + { + case PolicyLiteralExpression literal: + writer.WriteStartObject(); + writer.WriteString("type", "literal"); + writer.WritePropertyName("value"); + WriteLiteralValue(writer, literal.Value); + writer.WriteEndObject(); + break; + case PolicyListExpression list: + writer.WriteStartObject(); + writer.WriteString("type", "list"); + writer.WritePropertyName("items"); + writer.WriteStartArray(); + foreach (var item in list.Items) + { + WriteExpression(writer, item); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + break; + case PolicyIdentifierExpression identifier: + writer.WriteStartObject(); + writer.WriteString("type", "identifier"); + writer.WriteString("name", identifier.Name); + writer.WriteEndObject(); + break; + case PolicyMemberAccessExpression member: + writer.WriteStartObject(); + writer.WriteString("type", "member"); + writer.WritePropertyName("target"); + WriteExpression(writer, member.Target); + writer.WriteString("member", member.Member); + writer.WriteEndObject(); + break; + case PolicyInvocationExpression invocation: + writer.WriteStartObject(); + writer.WriteString("type", "call"); + writer.WritePropertyName("target"); + WriteExpression(writer, invocation.Target); + writer.WritePropertyName("args"); + writer.WriteStartArray(); + foreach (var arg in invocation.Arguments) + { + WriteExpression(writer, arg); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + break; + case PolicyIndexerExpression indexer: + writer.WriteStartObject(); + writer.WriteString("type", "indexer"); + writer.WritePropertyName("target"); + WriteExpression(writer, indexer.Target); + writer.WritePropertyName("index"); + WriteExpression(writer, indexer.Index); + writer.WriteEndObject(); + break; + case PolicyUnaryExpression unary: + writer.WriteStartObject(); + writer.WriteString("type", "unary"); + writer.WriteString("op", unary.Operator switch + { + PolicyUnaryOperator.Not => "not", + _ => unary.Operator.ToString().ToLowerInvariant(), + }); + writer.WritePropertyName("operand"); + WriteExpression(writer, unary.Operand); + writer.WriteEndObject(); + break; + case PolicyBinaryExpression binary: + writer.WriteStartObject(); + writer.WriteString("type", "binary"); + writer.WriteString("op", GetBinaryOperator(binary.Operator)); + writer.WritePropertyName("left"); + WriteExpression(writer, binary.Left); + writer.WritePropertyName("right"); + WriteExpression(writer, binary.Right); + writer.WriteEndObject(); + break; + default: + writer.WriteStartObject(); + writer.WriteString("type", "unknown"); + writer.WriteEndObject(); + break; + } + } + + private static string GetBinaryOperator(PolicyBinaryOperator op) => op switch + { + PolicyBinaryOperator.And => "and", + PolicyBinaryOperator.Or => "or", + PolicyBinaryOperator.Equal => "eq", + PolicyBinaryOperator.NotEqual => "neq", + PolicyBinaryOperator.LessThan => "lt", + PolicyBinaryOperator.LessThanOrEqual => "lte", + PolicyBinaryOperator.GreaterThan => "gt", + PolicyBinaryOperator.GreaterThanOrEqual => "gte", + PolicyBinaryOperator.In => "in", + PolicyBinaryOperator.NotIn => "not_in", + _ => op.ToString().ToLowerInvariant(), + }; + + private static void WriteLiteralDictionary(Utf8JsonWriter writer, ImmutableSortedDictionary dictionary) + { + writer.WriteStartObject(); + foreach (var kvp in dictionary) + { + writer.WritePropertyName(kvp.Key); + WriteLiteral(writer, kvp.Value); + } + + writer.WriteEndObject(); + } + + private static void WriteLiteral(Utf8JsonWriter writer, PolicyIrLiteral literal) + { + switch (literal) + { + case PolicyIrStringLiteral s: + writer.WriteStringValue(s.Value); + break; + case PolicyIrNumberLiteral n: + writer.WriteNumberValue(n.Value); + break; + case PolicyIrBooleanLiteral b: + writer.WriteBooleanValue(b.Value); + break; + case PolicyIrListLiteral list: + writer.WriteStartArray(); + foreach (var item in list.Items) + { + WriteLiteral(writer, item); + } + + writer.WriteEndArray(); + break; + default: + writer.WriteNullValue(); + break; + } + } + + private static void WriteLiteralValue(Utf8JsonWriter writer, object? value) + { + switch (value) + { + case null: + writer.WriteNullValue(); + break; + case string s: + writer.WriteStringValue(s); + break; + case bool b: + writer.WriteBooleanValue(b); + break; + case decimal dec: + writer.WriteNumberValue(dec); + break; + case double dbl: + writer.WriteNumberValue(dbl); + break; + case int i: + writer.WriteNumberValue(i); + break; + default: + writer.WriteStringValue(value.ToString()); + break; + } + } +} diff --git a/src/StellaOps.Policy.Engine/Compilation/PolicyParser.cs b/src/StellaOps.Policy.Engine/Compilation/PolicyParser.cs new file mode 100644 index 00000000..88158082 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/PolicyParser.cs @@ -0,0 +1,643 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using StellaOps.Policy; + +namespace StellaOps.Policy.Engine.Compilation; + +internal sealed class PolicyParser +{ + private readonly ImmutableArray tokens; + private readonly List diagnostics = new(); + private int position; + + private PolicyParser(ImmutableArray tokens) + { + this.tokens = tokens; + } + + public static PolicyParseResult Parse(string source) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + var tokenization = DslTokenizer.Tokenize(source); + var parser = new PolicyParser(tokenization.Tokens); + var document = parser.ParseDocument(); + var allDiagnostics = tokenization.Diagnostics.AddRange(parser.diagnostics).ToImmutableArray(); + return new PolicyParseResult(document, allDiagnostics); + } + + private PolicyDocumentNode? ParseDocument() + { + if (!Match(TokenKind.KeywordPolicy)) + { + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.MissingPolicyHeader, "Expected 'policy' declaration.", "policy")); + return null; + } + + var nameToken = Consume(TokenKind.StringLiteral, "Policy name must be a string literal.", "policy.name"); + var name = nameToken.Value as string ?? nameToken.Text; + + Consume(TokenKind.KeywordSyntax, "Expected 'syntax' declaration.", "policy.syntax"); + var syntaxToken = Consume(TokenKind.StringLiteral, "Policy syntax must be a string literal.", "policy.syntax.value"); + var syntax = syntaxToken.Value as string ?? syntaxToken.Text; + + Consume(TokenKind.LeftBrace, "Expected '{' to start policy body.", "policy.body"); + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + var settingsBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + var rules = ImmutableArray.CreateBuilder(); + + while (!Check(TokenKind.RightBrace) && !IsAtEnd) + { + if (Match(TokenKind.KeywordMetadata)) + { + foreach (var kvp in ParseKeyValueBlock("policy.metadata")) + { + metadataBuilder[kvp.Key] = kvp.Value; + } + + continue; + } + + if (Match(TokenKind.KeywordSettings)) + { + foreach (var kvp in ParseKeyValueBlock("policy.settings")) + { + settingsBuilder[kvp.Key] = kvp.Value; + } + + continue; + } + + if (Match(TokenKind.KeywordProfile)) + { + Consume(TokenKind.Identifier, "Profile requires a name.", "policy.profile"); + Consume(TokenKind.LeftBrace, "Expected '{' after profile declaration.", "policy.profile"); + SkipBlock(); + continue; + } + + if (Match(TokenKind.KeywordRule)) + { + var rule = ParseRule(); + if (rule is not null) + { + rules.Add(rule); + } + + continue; + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnexpectedSection, $"Unexpected token '{Current.Text}' in policy body.", "policy.body")); + Advance(); + } + + var close = Consume(TokenKind.RightBrace, "Expected '}' to close policy definition.", "policy"); + + if (!string.Equals(syntax, "stella-dsl@1", StringComparison.Ordinal)) + { + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnsupportedSyntaxVersion, $"Unsupported syntax '{syntax}'.", "policy.syntax")); + } + + var span = new SourceSpan(tokens[0].Span.Start, close.Span.End); + return new PolicyDocumentNode( + name, + syntax, + metadataBuilder.ToImmutable(), + ImmutableArray.Empty, + settingsBuilder.ToImmutable(), + rules.ToImmutable(), + span); + } + + private PolicyRuleNode? ParseRule() + { + var nameToken = Consume(TokenKind.Identifier, "Rule requires a name.", "policy.rule"); + var name = nameToken.Text; + + var priority = 0; + if (Match(TokenKind.KeywordPriority)) + { + var priorityToken = Consume(TokenKind.NumberLiteral, "Priority must be numeric.", $"policy.rule.{name}"); + if (priorityToken.Value is decimal dec) + { + priority = (int)Math.Round(dec, MidpointRounding.AwayFromZero); + } + } + + Consume(TokenKind.LeftBrace, "Expected '{' to start rule.", $"policy.rule.{name}"); + Consume(TokenKind.KeywordWhen, "Rule requires a 'when' clause.", $"policy.rule.{name}"); + var when = ParseExpression(); + + Consume(TokenKind.KeywordThen, "Rule requires a 'then' clause.", $"policy.rule.{name}"); + var thenActions = ParseActions(name, "then"); + + var elseActions = ImmutableArray.Empty; + if (Match(TokenKind.KeywordElse)) + { + elseActions = ParseActions(name, "else"); + } + + string? because = null; + if (Match(TokenKind.KeywordBecause)) + { + var becauseToken = Consume(TokenKind.StringLiteral, "Because clause must be string.", $"policy.rule.{name}.because"); + because = becauseToken.Value as string ?? becauseToken.Text; + } + + var close = Consume(TokenKind.RightBrace, "Expected '}' to close rule.", $"policy.rule.{name}"); + + if (because is null) + { + diagnostics.Add(PolicyIssue.Warning(PolicyDslDiagnosticCodes.MissingBecauseClause, $"Rule '{name}' missing 'because' clause.", $"policy.rule.{name}")); + } + + return new PolicyRuleNode(name, priority, when, thenActions, elseActions, because, new SourceSpan(nameToken.Span.Start, close.Span.End)); + } + + private ImmutableArray ParseActions(string ruleName, string clause) + { + var actions = ImmutableArray.CreateBuilder(); + while (!Check(TokenKind.RightBrace) && !Check(TokenKind.KeywordElse) && !Check(TokenKind.KeywordBecause) && !IsAtEnd) + { + if (Check(TokenKind.Identifier)) + { + actions.Add(ParseAssignmentAction(ruleName, clause)); + continue; + } + + if (Match(TokenKind.KeywordAnnotate)) + { + actions.Add(ParseAnnotateAction(ruleName, clause)); + continue; + } + + if (Match(TokenKind.KeywordWarn)) + { + actions.Add(ParseWarnAction()); + continue; + } + + if (Match(TokenKind.KeywordEscalate)) + { + actions.Add(ParseEscalateAction()); + continue; + } + + if (Match(TokenKind.KeywordRequireVex)) + { + actions.Add(ParseRequireVexAction(ruleName, clause)); + continue; + } + + if (Match(TokenKind.KeywordIgnore)) + { + actions.Add(ParseIgnoreAction(ruleName, clause)); + continue; + } + + if (Match(TokenKind.KeywordDefer)) + { + actions.Add(ParseDeferAction(ruleName, clause)); + continue; + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.InvalidAction, $"Unexpected token '{Current.Text}' in {clause} actions.", $"policy.rule.{ruleName}.{clause}")); + Advance(); + } + + return actions.ToImmutable(); + } + + private PolicyActionNode ParseAssignmentAction(string ruleName, string clause) + { + var segments = ImmutableArray.CreateBuilder(); + var first = Consume(TokenKind.Identifier, "Assignment target must start with identifier.", $"policy.rule.{ruleName}.{clause}"); + segments.Add(first.Text); + while (Match(TokenKind.Dot)) + { + segments.Add(Consume(TokenKind.Identifier, "Expected identifier after '.'.", $"policy.rule.{ruleName}.{clause}").Text); + } + + Consume(TokenKind.Define, "Expected ':=' in action.", $"policy.rule.{ruleName}.{clause}"); + var value = ParseExpression(); + Match(TokenKind.Semicolon); + return new PolicyAssignmentActionNode(new PolicyReference(segments.ToImmutable(), new SourceSpan(first.Span.Start, value.Span.End)), value, new SourceSpan(first.Span.Start, value.Span.End)); + } + + private PolicyActionNode ParseAnnotateAction(string ruleName, string clause) + { + var reference = ParseReference($"policy.rule.{ruleName}.{clause}.annotate"); + Consume(TokenKind.Define, "Expected ':=' in annotate action.", $"policy.rule.{ruleName}.{clause}.annotate"); + var value = ParseExpression(); + Match(TokenKind.Semicolon); + return new PolicyAnnotateActionNode(reference, value, new SourceSpan(reference.Span.Start, value.Span.End)); + } + + private PolicyActionNode ParseWarnAction() + { + PolicyExpression? message = null; + if (Match(TokenKind.KeywordMessage)) + { + message = ParseExpression(); + } + + Match(TokenKind.Semicolon); + var span = message?.Span ?? Previous.Span; + return new PolicyWarnActionNode(message, span); + } + + private PolicyActionNode ParseEscalateAction() + { + PolicyExpression? to = null; + PolicyExpression? when = null; + + if (Match(TokenKind.KeywordTo)) + { + to = ParseExpression(); + } + + if (Match(TokenKind.KeywordWhen)) + { + when = ParseExpression(); + } + + Match(TokenKind.Semicolon); + var end = when?.Span.End ?? to?.Span.End ?? Previous.Span.End; + return new PolicyEscalateActionNode(to, when, new SourceSpan(Previous.Span.Start, end)); + } + + private PolicyActionNode ParseRequireVexAction(string ruleName, string clause) + { + Consume(TokenKind.LeftBrace, "Expected '{' after requireVex.", $"policy.rule.{ruleName}.{clause}.requireVex"); + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + while (!Check(TokenKind.RightBrace) && !IsAtEnd) + { + var key = Consume(TokenKind.Identifier, "requireVex key must be identifier.", $"policy.rule.{ruleName}.{clause}.requireVex").Text; + Consume(TokenKind.Assign, "Expected '=' in requireVex condition.", $"policy.rule.{ruleName}.{clause}.requireVex"); + builder[key] = ParseExpression(); + Match(TokenKind.Comma); + } + + var close = Consume(TokenKind.RightBrace, "Expected '}' to close requireVex block.", $"policy.rule.{ruleName}.{clause}.requireVex"); + Match(TokenKind.Semicolon); + return new PolicyRequireVexActionNode(builder.ToImmutable(), new SourceSpan(close.Span.Start, close.Span.End)); + } + + private PolicyActionNode ParseIgnoreAction(string ruleName, string clause) + { + PolicyExpression? until = null; + string? because = null; + if (Match(TokenKind.KeywordUntil)) + { + until = ParseExpression(); + } + + if (Match(TokenKind.KeywordBecause)) + { + var becauseToken = Consume(TokenKind.StringLiteral, "Ignore 'because' must be string.", $"policy.rule.{ruleName}.{clause}.ignore"); + because = becauseToken.Value as string ?? becauseToken.Text; + } + + Match(TokenKind.Semicolon); + return new PolicyIgnoreActionNode(until, because, new SourceSpan(Previous.Span.Start, (until?.Span.End ?? Previous.Span.End))); + } + + private PolicyActionNode ParseDeferAction(string ruleName, string clause) + { + PolicyExpression? until = null; + if (Match(TokenKind.KeywordUntil)) + { + until = ParseExpression(); + } + + Match(TokenKind.Semicolon); + return new PolicyDeferActionNode(until, new SourceSpan(Previous.Span.Start, (until?.Span.End ?? Previous.Span.End))); + } + + private PolicyReference ParseReference(string path) + { + var segments = ImmutableArray.CreateBuilder(); + var first = Consume(TokenKind.Identifier, "Expected identifier.", path); + segments.Add(first.Text); + while (Match(TokenKind.Dot)) + { + segments.Add(Consume(TokenKind.Identifier, "Expected identifier after '.'.", path).Text); + } + + return new PolicyReference(segments.ToImmutable(), first.Span); + } + + private Dictionary ParseKeyValueBlock(string path) + { + Consume(TokenKind.LeftBrace, "Expected '{'.", path); + var entries = new Dictionary(StringComparer.Ordinal); + while (!Check(TokenKind.RightBrace) && !IsAtEnd) + { + var key = Consume(TokenKind.Identifier, "Expected identifier.", path).Text; + Consume(TokenKind.Assign, "Expected '='.", path); + entries[key] = ParseLiteralValue(path); + Match(TokenKind.Semicolon); + } + + Consume(TokenKind.RightBrace, "Expected '}'.", path); + return entries; + } + + private PolicyLiteralValue ParseLiteralValue(string path) + { + if (Match(TokenKind.StringLiteral)) + { + return new PolicyStringLiteral(Previous.Value as string ?? Previous.Text, Previous.Span); + } + + if (Match(TokenKind.NumberLiteral)) + { + return new PolicyNumberLiteral(Previous.Value is decimal dec ? dec : 0m, Previous.Span); + } + + if (Match(TokenKind.BooleanLiteral)) + { + return new PolicyBooleanLiteral(Previous.Value is bool b && b, Previous.Span); + } + + if (Match(TokenKind.LeftBracket)) + { + var start = Previous.Span.Start; + var items = ImmutableArray.CreateBuilder(); + while (!Check(TokenKind.RightBracket) && !IsAtEnd) + { + items.Add(ParseLiteralValue(path)); + Match(TokenKind.Comma); + } + + var close = Consume(TokenKind.RightBracket, "Expected ']' in list literal.", path); + return new PolicyListLiteral(items.ToImmutable(), new SourceSpan(start, close.Span.End)); + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.InvalidLiteral, "Invalid literal.", path)); + return new PolicyStringLiteral(string.Empty, Current.Span); + } + + private PolicyExpression ParseExpression() => ParseOr(); + + private PolicyExpression ParseOr() + { + var expr = ParseAnd(); + while (Match(TokenKind.KeywordOr)) + { + var right = ParseAnd(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.Or, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + + return expr; + } + + private PolicyExpression ParseAnd() + { + var expr = ParseEquality(); + while (Match(TokenKind.KeywordAnd)) + { + var right = ParseEquality(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.And, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + + return expr; + } + + private PolicyExpression ParseEquality() + { + var expr = ParseUnary(); + while (true) + { + if (Match(TokenKind.EqualEqual)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.Equal, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else if (Match(TokenKind.NotEqual)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.NotEqual, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else if (Match(TokenKind.KeywordIn)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.In, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else if (Match(TokenKind.KeywordNot)) + { + if (Match(TokenKind.KeywordIn)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.NotIn, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else + { + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnexpectedToken, "Expected 'in' after 'not'.", "expression.not")); + } + } + else if (Match(TokenKind.LessThan)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.LessThan, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else if (Match(TokenKind.LessThanOrEqual)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.LessThanOrEqual, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else if (Match(TokenKind.GreaterThan)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.GreaterThan, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else if (Match(TokenKind.GreaterThanOrEqual)) + { + var right = ParseUnary(); + expr = new PolicyBinaryExpression(expr, PolicyBinaryOperator.GreaterThanOrEqual, right, new SourceSpan(expr.Span.Start, right.Span.End)); + } + else + { + break; + } + } + + return expr; + } + + private PolicyExpression ParseUnary() + { + if (Match(TokenKind.KeywordNot)) + { + var operand = ParseUnary(); + return new PolicyUnaryExpression(PolicyUnaryOperator.Not, operand, new SourceSpan(Previous.Span.Start, operand.Span.End)); + } + + return ParsePrimary(); + } + + private PolicyExpression ParsePrimary() + { + if (Match(TokenKind.StringLiteral)) + { + return new PolicyLiteralExpression(Previous.Value as string ?? Previous.Text, Previous.Span); + } + + if (Match(TokenKind.NumberLiteral)) + { + return new PolicyLiteralExpression(Previous.Value ?? 0m, Previous.Span); + } + + if (Match(TokenKind.BooleanLiteral)) + { + return new PolicyLiteralExpression(Previous.Value ?? false, Previous.Span); + } + + if (Match(TokenKind.LeftBracket)) + { + var start = Previous.Span.Start; + var items = ImmutableArray.CreateBuilder(); + while (!Check(TokenKind.RightBracket) && !IsAtEnd) + { + items.Add(ParseExpression()); + Match(TokenKind.Comma); + } + + var close = Consume(TokenKind.RightBracket, "Expected ']' to close list expression.", "expression.list"); + return new PolicyListExpression(items.ToImmutable(), new SourceSpan(start, close.Span.End)); + } + + if (Match(TokenKind.LeftParen)) + { + var expr = ParseExpression(); + Consume(TokenKind.RightParen, "Expected ')' to close grouped expression.", "expression.group"); + return expr; + } + + if (Match(TokenKind.Identifier)) + { + return ParseIdentifierExpression(Previous); + } + + if (Match(TokenKind.KeywordEnv)) + { + return ParseIdentifierExpression(Previous); + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnexpectedToken, $"Unexpected token '{Current.Text}' in expression.", "expression")); + var bad = Advance(); + return new PolicyLiteralExpression(null, bad.Span); + } + + private PolicyExpression ParseIdentifierExpression(DslToken identifier) + { + PolicyExpression expr = new PolicyIdentifierExpression(identifier.Text, identifier.Span); + while (true) + { + if (Match(TokenKind.Dot)) + { + var member = Consume(TokenKind.Identifier, "Expected identifier after '.'.", "expression.member"); + expr = new PolicyMemberAccessExpression(expr, member.Text, new SourceSpan(expr.Span.Start, member.Span.End)); + continue; + } + + if (Match(TokenKind.LeftParen)) + { + var args = ImmutableArray.CreateBuilder(); + if (!Check(TokenKind.RightParen)) + { + do + { + args.Add(ParseExpression()); + } + while (Match(TokenKind.Comma)); + } + + var close = Consume(TokenKind.RightParen, "Expected ')' to close invocation.", "expression.call"); + expr = new PolicyInvocationExpression(expr, args.ToImmutable(), new SourceSpan(expr.Span.Start, close.Span.End)); + continue; + } + + if (Match(TokenKind.LeftBracket)) + { + var indexExpr = ParseExpression(); + var close = Consume(TokenKind.RightBracket, "Expected ']' to close indexer.", "expression.indexer"); + expr = new PolicyIndexerExpression(expr, indexExpr, new SourceSpan(expr.Span.Start, close.Span.End)); + continue; + } + + break; + } + + return expr; + } + + private bool Match(TokenKind kind) + { + if (Check(kind)) + { + Advance(); + return true; + } + + return false; + } + + private bool Check(TokenKind kind) => !IsAtEnd && Current.Kind == kind; + + private DslToken Consume(TokenKind kind, string message, string path) + { + if (Check(kind)) + { + return Advance(); + } + + diagnostics.Add(PolicyIssue.Error(PolicyDslDiagnosticCodes.UnexpectedToken, message, path)); + return Advance(); + } + + private void SkipBlock() + { + var depth = 1; + while (depth > 0 && !IsAtEnd) + { + if (Match(TokenKind.LeftBrace)) + { + depth++; + } + else if (Match(TokenKind.RightBrace)) + { + depth--; + } + else + { + Advance(); + } + } + } + + private DslToken Advance() + { + if (!IsAtEnd) + { + position++; + } + + return tokens[position - 1]; + } + + private bool IsAtEnd => Current.Kind == TokenKind.EndOfFile; + + private DslToken Current => tokens[position]; + + private DslToken Previous => tokens[position - 1]; +} + +internal readonly record struct PolicyParseResult( + PolicyDocumentNode? Document, + ImmutableArray Diagnostics); diff --git a/src/StellaOps.Policy.Engine/Compilation/PolicySyntaxNodes.cs b/src/StellaOps.Policy.Engine/Compilation/PolicySyntaxNodes.cs new file mode 100644 index 00000000..dc815935 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Compilation/PolicySyntaxNodes.cs @@ -0,0 +1,141 @@ +using System.Collections.Immutable; + +namespace StellaOps.Policy.Engine.Compilation; + +public abstract record SyntaxNode(SourceSpan Span); + +public sealed record PolicyDocumentNode( + string Name, + string Syntax, + ImmutableDictionary Metadata, + ImmutableArray Profiles, + ImmutableDictionary Settings, + ImmutableArray Rules, + SourceSpan Span) : SyntaxNode(Span); + +public sealed record PolicyProfileNode( + string Name, + ImmutableArray Items, + SourceSpan Span) : SyntaxNode(Span); + +public abstract record PolicyProfileItemNode(SourceSpan Span); + +public sealed record PolicyProfileMapNode( + string Name, + ImmutableArray Entries, + SourceSpan Span) : PolicyProfileItemNode(Span); + +public sealed record PolicyProfileMapEntryNode( + string Source, + decimal Weight, + SourceSpan Span) : SyntaxNode(Span); + +public sealed record PolicyProfileEnvNode( + string Name, + ImmutableArray Entries, + SourceSpan Span) : PolicyProfileItemNode(Span); + +public sealed record PolicyProfileEnvEntryNode( + PolicyExpression Condition, + decimal Weight, + SourceSpan Span) : SyntaxNode(Span); + +public sealed record PolicyProfileScalarNode( + string Name, + PolicyLiteralValue Value, + SourceSpan Span) : PolicyProfileItemNode(Span); + +public sealed record PolicyRuleNode( + string Name, + int Priority, + PolicyExpression When, + ImmutableArray ThenActions, + ImmutableArray ElseActions, + string? Because, + SourceSpan Span) : SyntaxNode(Span); + +public abstract record PolicyActionNode(SourceSpan Span); + +public sealed record PolicyAssignmentActionNode( + PolicyReference Target, + PolicyExpression Value, + SourceSpan Span) : PolicyActionNode(Span); + +public sealed record PolicyAnnotateActionNode( + PolicyReference Target, + PolicyExpression Value, + SourceSpan Span) : PolicyActionNode(Span); + +public sealed record PolicyIgnoreActionNode( + PolicyExpression? Until, + string? Because, + SourceSpan Span) : PolicyActionNode(Span); + +public sealed record PolicyEscalateActionNode( + PolicyExpression? To, + PolicyExpression? When, + SourceSpan Span) : PolicyActionNode(Span); + +public sealed record PolicyRequireVexActionNode( + ImmutableDictionary Conditions, + SourceSpan Span) : PolicyActionNode(Span); + +public sealed record PolicyWarnActionNode( + PolicyExpression? Message, + SourceSpan Span) : PolicyActionNode(Span); + +public sealed record PolicyDeferActionNode( + PolicyExpression? Until, + SourceSpan Span) : PolicyActionNode(Span); + +public abstract record PolicyExpression(SourceSpan Span); + +public sealed record PolicyLiteralExpression(object? Value, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyListExpression(ImmutableArray Items, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyIdentifierExpression(string Name, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyMemberAccessExpression(PolicyExpression Target, string Member, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyInvocationExpression(PolicyExpression Target, ImmutableArray Arguments, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyIndexerExpression(PolicyExpression Target, PolicyExpression Index, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyUnaryExpression(PolicyUnaryOperator Operator, PolicyExpression Operand, SourceSpan Span) : PolicyExpression(Span); + +public sealed record PolicyBinaryExpression(PolicyExpression Left, PolicyBinaryOperator Operator, PolicyExpression Right, SourceSpan Span) : PolicyExpression(Span); + +public enum PolicyUnaryOperator +{ + Not, +} + +public enum PolicyBinaryOperator +{ + And, + Or, + Equal, + NotEqual, + LessThan, + LessThanOrEqual, + GreaterThan, + GreaterThanOrEqual, + In, + NotIn, +} + +public sealed record PolicyReference(ImmutableArray Segments, SourceSpan Span) +{ + public override string ToString() => string.Join(".", Segments); +} + +public abstract record PolicyLiteralValue(SourceSpan Span); + +public sealed record PolicyStringLiteral(string Value, SourceSpan Span) : PolicyLiteralValue(Span); + +public sealed record PolicyNumberLiteral(decimal Value, SourceSpan Span) : PolicyLiteralValue(Span); + +public sealed record PolicyBooleanLiteral(bool Value, SourceSpan Span) : PolicyLiteralValue(Span); + +public sealed record PolicyListLiteral(ImmutableArray Items, SourceSpan Span) : PolicyLiteralValue(Span); diff --git a/src/StellaOps.Policy.Engine/Endpoints/PolicyCompilationEndpoints.cs b/src/StellaOps.Policy.Engine/Endpoints/PolicyCompilationEndpoints.cs new file mode 100644 index 00000000..2c74c4b9 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Endpoints/PolicyCompilationEndpoints.cs @@ -0,0 +1,107 @@ +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.AspNetCore.Mvc; +using StellaOps.Policy; +using StellaOps.Policy.Engine.Services; +using System.Collections.Immutable; + +namespace StellaOps.Policy.Engine.Endpoints; + +internal static class PolicyCompilationEndpoints +{ + private const string CompileRoute = "/api/policy/policies/{policyId}/versions/{version}:compile"; + + public static IEndpointRouteBuilder MapPolicyCompilation(this IEndpointRouteBuilder endpoints) + { + endpoints.MapPost(CompileRoute, CompilePolicy) + .WithName("CompilePolicy") + .WithSummary("Compile and lint a policy DSL document.") + .WithDescription("Compiles a stella-dsl@1 policy document and returns deterministic digest and statistics.") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(); // scopes enforced by policy middleware. + + return endpoints; + } + + private static IResult CompilePolicy( + [FromRoute] string policyId, + [FromRoute] int version, + [FromBody] PolicyCompileRequest request, + PolicyCompilationService compilationService) + { + if (request is null) + { + return Results.BadRequest(BuildProblem("ERR_POL_001", "Request body missing.", policyId, version)); + } + + var result = compilationService.Compile(request); + if (!result.Success) + { + return Results.BadRequest(BuildProblem("ERR_POL_001", "Policy compilation failed.", policyId, version, result.Diagnostics)); + } + + var response = new PolicyCompileResponse( + result.Digest!, + result.Statistics ?? new PolicyCompilationStatistics(0, ImmutableDictionary.Empty), + ConvertDiagnostics(result.Diagnostics)); + return Results.Ok(response); + } + + private static PolicyProblemDetails BuildProblem(string code, string message, string policyId, int version, ImmutableArray? diagnostics = null) + { + var problem = new PolicyProblemDetails + { + Code = code, + Title = "Policy compilation error", + Detail = message, + PolicyId = policyId, + PolicyVersion = version + }; + + if (diagnostics is { Length: > 0 } diag) + { + problem.Diagnostics = diag; + } + + return problem; + } + + private static ImmutableArray ConvertDiagnostics(ImmutableArray issues) + { + if (issues.IsDefaultOrEmpty) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(issues.Length); + foreach (var issue in issues) + { + if (issue.Severity != PolicyIssueSeverity.Warning) + { + continue; + } + + builder.Add(new PolicyDiagnosticDto(issue.Code, issue.Message, issue.Path)); + } + + return builder.ToImmutable(); + } + + private sealed class PolicyProblemDetails : ProblemDetails + { + public string Code { get; set; } = "ERR_POL_001"; + + public string? PolicyId { get; set; } + + public int PolicyVersion { get; set; } + + public ImmutableArray Diagnostics { get; set; } = ImmutableArray.Empty; + } +} + +internal sealed record PolicyCompileResponse( + string Digest, + PolicyCompilationStatistics Statistics, + ImmutableArray Warnings); + +internal sealed record PolicyDiagnosticDto(string Code, string Message, string Path); diff --git a/src/StellaOps.Policy.Engine/Evaluation/PolicyEvaluationContext.cs b/src/StellaOps.Policy.Engine/Evaluation/PolicyEvaluationContext.cs new file mode 100644 index 00000000..525526eb --- /dev/null +++ b/src/StellaOps.Policy.Engine/Evaluation/PolicyEvaluationContext.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Immutable; +using StellaOps.Policy.Engine.Compilation; + +namespace StellaOps.Policy.Engine.Evaluation; + +internal sealed record PolicyEvaluationRequest( + PolicyIrDocument Document, + PolicyEvaluationContext Context); + +internal sealed record PolicyEvaluationContext( + PolicyEvaluationSeverity Severity, + PolicyEvaluationEnvironment Environment, + PolicyEvaluationAdvisory Advisory, + PolicyEvaluationVexEvidence Vex, + PolicyEvaluationSbom Sbom); + +internal sealed record PolicyEvaluationSeverity(string Normalized, decimal? Score = null); + +internal sealed record PolicyEvaluationEnvironment( + ImmutableDictionary Properties) +{ + public string? Get(string key) => Properties.TryGetValue(key, out var value) ? value : null; +} + +internal sealed record PolicyEvaluationAdvisory( + string Source, + ImmutableDictionary Metadata); + +internal sealed record PolicyEvaluationVexEvidence( + ImmutableArray Statements) +{ + public static readonly PolicyEvaluationVexEvidence Empty = new(ImmutableArray.Empty); +} + +internal sealed record PolicyEvaluationVexStatement( + string Status, + string Justification, + string StatementId, + DateTimeOffset? Timestamp = null); + +internal sealed record PolicyEvaluationSbom(ImmutableHashSet Tags) +{ + public bool HasTag(string tag) => Tags.Contains(tag); +} + +internal sealed record PolicyEvaluationResult( + bool Matched, + string Status, + string? Severity, + string? RuleName, + int? Priority, + ImmutableDictionary Annotations, + ImmutableArray Warnings) +{ + public static PolicyEvaluationResult CreateDefault(string? severity) => new( + Matched: false, + Status: "affected", + Severity: severity, + RuleName: null, + Priority: null, + Annotations: ImmutableDictionary.Empty, + Warnings: ImmutableArray.Empty); +} diff --git a/src/StellaOps.Policy.Engine/Evaluation/PolicyEvaluator.cs b/src/StellaOps.Policy.Engine/Evaluation/PolicyEvaluator.cs new file mode 100644 index 00000000..555afe74 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Evaluation/PolicyEvaluator.cs @@ -0,0 +1,184 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Policy.Engine.Compilation; + +namespace StellaOps.Policy.Engine.Evaluation; + +/// +/// Deterministically evaluates compiled policy IR against advisory/VEX/SBOM inputs. +/// +internal sealed class PolicyEvaluator +{ + public PolicyEvaluationResult Evaluate(PolicyEvaluationRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (request.Document is null) + { + throw new ArgumentNullException(nameof(request.Document)); + } + + var evaluator = new PolicyExpressionEvaluator(request.Context); + var orderedRules = request.Document.Rules + .Select(static (rule, index) => new { rule, index }) + .OrderBy(x => x.rule.Priority) + .ThenBy(x => x.index) + .ToImmutableArray(); + + foreach (var entry in orderedRules) + { + var rule = entry.rule; + if (!evaluator.EvaluateBoolean(rule.When)) + { + continue; + } + + var runtime = new PolicyRuntimeState(request.Context.Severity.Normalized); + foreach (var action in rule.ThenActions) + { + ApplyAction(rule.Name, action, evaluator, runtime); + } + + if (runtime.Status is null) + { + runtime.Status = "affected"; + } + + return new PolicyEvaluationResult( + Matched: true, + Status: runtime.Status, + Severity: runtime.Severity, + RuleName: rule.Name, + Priority: rule.Priority, + Annotations: runtime.Annotations.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase), + Warnings: runtime.Warnings.ToImmutableArray()); + } + + return PolicyEvaluationResult.CreateDefault(request.Context.Severity.Normalized); + } + + private static void ApplyAction( + string ruleName, + PolicyIrAction action, + PolicyExpressionEvaluator evaluator, + PolicyRuntimeState runtime) + { + switch (action) + { + case PolicyIrAssignmentAction assign: + ApplyAssignment(assign, evaluator, runtime); + break; + case PolicyIrAnnotateAction annotate: + ApplyAnnotate(annotate, evaluator, runtime); + break; + case PolicyIrWarnAction warn: + ApplyWarn(warn, evaluator, runtime); + break; + case PolicyIrEscalateAction escalate: + ApplyEscalate(escalate, evaluator, runtime); + break; + case PolicyIrRequireVexAction require: + var allSatisfied = true; + foreach (var condition in require.Conditions.Values) + { + if (!evaluator.EvaluateBoolean(condition)) + { + allSatisfied = false; + break; + } + } + + runtime.Status ??= allSatisfied ? "affected" : "suppressed"; + break; + case PolicyIrIgnoreAction ignore: + runtime.Status = "ignored"; + break; + case PolicyIrDeferAction defer: + runtime.Status = "deferred"; + break; + default: + runtime.Warnings.Add($"Unhandled action '{action.GetType().Name}' in rule '{ruleName}'."); + break; + } + } + + private static void ApplyAssignment(PolicyIrAssignmentAction assign, PolicyExpressionEvaluator evaluator, PolicyRuntimeState runtime) + { + var value = evaluator.Evaluate(assign.Value); + var stringValue = value.AsString(); + if (assign.Target.Length == 0) + { + return; + } + + var target = assign.Target[0]; + switch (target) + { + case "status": + runtime.Status = stringValue ?? runtime.Status ?? "affected"; + break; + case "severity": + runtime.Severity = stringValue; + break; + default: + runtime.Annotations[target] = stringValue ?? value.Raw?.ToString() ?? string.Empty; + break; + } + } + + private static void ApplyAnnotate(PolicyIrAnnotateAction annotate, PolicyExpressionEvaluator evaluator, PolicyRuntimeState runtime) + { + var key = annotate.Target.Length > 0 ? annotate.Target[^1] : "annotation"; + var value = evaluator.Evaluate(annotate.Value).AsString() ?? string.Empty; + runtime.Annotations[key] = value; + } + + private static void ApplyWarn(PolicyIrWarnAction warn, PolicyExpressionEvaluator evaluator, PolicyRuntimeState runtime) + { + var message = warn.Message is null ? "" : evaluator.Evaluate(warn.Message).AsString(); + if (!string.IsNullOrWhiteSpace(message)) + { + runtime.Warnings.Add(message!); + } + else + { + runtime.Warnings.Add("Policy rule emitted a warning."); + } + + runtime.Status ??= "warned"; + } + + private static void ApplyEscalate(PolicyIrEscalateAction escalate, PolicyExpressionEvaluator evaluator, PolicyRuntimeState runtime) + { + if (escalate.To is not null) + { + runtime.Severity = evaluator.Evaluate(escalate.To).AsString() ?? runtime.Severity; + } + + if (escalate.When is not null && !evaluator.EvaluateBoolean(escalate.When)) + { + return; + } + } + + private sealed class PolicyRuntimeState + { + public PolicyRuntimeState(string? initialSeverity) + { + Severity = initialSeverity; + } + + public string? Status { get; set; } + + public string? Severity { get; set; } + + public Dictionary Annotations { get; } = new(StringComparer.OrdinalIgnoreCase); + + public List Warnings { get; } = new(); + } +} diff --git a/src/StellaOps.Policy.Engine/Evaluation/PolicyExpressionEvaluator.cs b/src/StellaOps.Policy.Engine/Evaluation/PolicyExpressionEvaluator.cs new file mode 100644 index 00000000..0e5816b5 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Evaluation/PolicyExpressionEvaluator.cs @@ -0,0 +1,470 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using StellaOps.Policy.Engine.Compilation; + +namespace StellaOps.Policy.Engine.Evaluation; + +internal sealed class PolicyExpressionEvaluator +{ + private readonly PolicyEvaluationContext context; + + public PolicyExpressionEvaluator(PolicyEvaluationContext context) + { + this.context = context ?? throw new ArgumentNullException(nameof(context)); + } + + public EvaluationValue Evaluate(PolicyExpression expression, EvaluationScope? scope = null) + { + scope ??= EvaluationScope.Root(context); + return expression switch + { + PolicyLiteralExpression literal => new EvaluationValue(literal.Value), + PolicyListExpression list => new EvaluationValue(list.Items.Select(item => Evaluate(item, scope).Raw).ToImmutableArray()), + PolicyIdentifierExpression identifier => ResolveIdentifier(identifier.Name, scope), + PolicyMemberAccessExpression member => EvaluateMember(member, scope), + PolicyInvocationExpression invocation => EvaluateInvocation(invocation, scope), + PolicyIndexerExpression indexer => EvaluateIndexer(indexer, scope), + PolicyUnaryExpression unary => EvaluateUnary(unary, scope), + PolicyBinaryExpression binary => EvaluateBinary(binary, scope), + _ => EvaluationValue.Null, + }; + } + + public bool EvaluateBoolean(PolicyExpression expression, EvaluationScope? scope = null) => + Evaluate(expression, scope).AsBoolean(); + + private EvaluationValue ResolveIdentifier(string name, EvaluationScope scope) + { + if (scope.TryGetLocal(name, out var local)) + { + return new EvaluationValue(local); + } + + return name switch + { + "severity" => new EvaluationValue(new SeverityScope(context.Severity)), + "env" => new EvaluationValue(new EnvironmentScope(context.Environment)), + "vex" => new EvaluationValue(new VexScope(this, context.Vex)), + "advisory" => new EvaluationValue(new AdvisoryScope(context.Advisory)), + "sbom" => new EvaluationValue(new SbomScope(context.Sbom)), + "true" => EvaluationValue.True, + "false" => EvaluationValue.False, + _ => EvaluationValue.Null, + }; + } + + private EvaluationValue EvaluateMember(PolicyMemberAccessExpression member, EvaluationScope scope) + { + var target = Evaluate(member.Target, scope); + var raw = target.Raw; + if (raw is SeverityScope severity) + { + return severity.Get(member.Member); + } + + if (raw is EnvironmentScope env) + { + return env.Get(member.Member); + } + + if (raw is VexScope vex) + { + return vex.Get(member.Member); + } + + if (raw is AdvisoryScope advisory) + { + return advisory.Get(member.Member); + } + + if (raw is SbomScope sbom) + { + return sbom.Get(member.Member); + } + + if (raw is ImmutableDictionary dict && dict.TryGetValue(member.Member, out var value)) + { + return new EvaluationValue(value); + } + + if (raw is PolicyEvaluationVexStatement stmt) + { + return member.Member switch + { + "status" => new EvaluationValue(stmt.Status), + "justification" => new EvaluationValue(stmt.Justification), + "statementId" => new EvaluationValue(stmt.StatementId), + _ => EvaluationValue.Null, + }; + } + + return EvaluationValue.Null; + } + + private EvaluationValue EvaluateInvocation(PolicyInvocationExpression invocation, EvaluationScope scope) + { + if (invocation.Target is PolicyIdentifierExpression identifier) + { + switch (identifier.Name) + { + case "severity_band": + var arg = invocation.Arguments.Length > 0 ? Evaluate(invocation.Arguments[0], scope).AsString() : null; + return new EvaluationValue(arg ?? string.Empty); + } + } + + if (invocation.Target is PolicyMemberAccessExpression member && member.Target is PolicyIdentifierExpression root) + { + if (root.Name == "vex") + { + var vex = Evaluate(member.Target, scope); + if (vex.Raw is VexScope vexScope) + { + return member.Member switch + { + "any" => new EvaluationValue(vexScope.Any(invocation.Arguments, scope)), + "latest" => new EvaluationValue(vexScope.Latest()), + _ => EvaluationValue.Null, + }; + } + } + + if (root.Name == "sbom") + { + var sbom = Evaluate(member.Target, scope); + if (sbom.Raw is SbomScope sbomScope) + { + return member.Member switch + { + "has_tag" => sbomScope.HasTag(invocation.Arguments, scope, this), + _ => EvaluationValue.Null, + }; + } + } + + if (root.Name == "advisory") + { + var advisory = Evaluate(member.Target, scope); + if (advisory.Raw is AdvisoryScope advisoryScope) + { + return advisoryScope.Invoke(member.Member, invocation.Arguments, scope, this); + } + } + } + + return EvaluationValue.Null; + } + + private EvaluationValue EvaluateIndexer(PolicyIndexerExpression indexer, EvaluationScope scope) + { + var target = Evaluate(indexer.Target, scope).Raw; + var index = Evaluate(indexer.Index, scope).Raw; + + if (target is ImmutableArray array && index is int i && i >= 0 && i < array.Length) + { + return new EvaluationValue(array[i]); + } + + return EvaluationValue.Null; + } + + private EvaluationValue EvaluateUnary(PolicyUnaryExpression unary, EvaluationScope scope) + { + var operand = Evaluate(unary.Operand, scope); + return unary.Operator switch + { + PolicyUnaryOperator.Not => new EvaluationValue(!operand.AsBoolean()), + _ => EvaluationValue.Null, + }; + } + + private EvaluationValue EvaluateBinary(PolicyBinaryExpression binary, EvaluationScope scope) + { + return binary.Operator switch + { + PolicyBinaryOperator.And => new EvaluationValue(EvaluateBoolean(binary.Left, scope) && EvaluateBoolean(binary.Right, scope)), + PolicyBinaryOperator.Or => new EvaluationValue(EvaluateBoolean(binary.Left, scope) || EvaluateBoolean(binary.Right, scope)), + PolicyBinaryOperator.Equal => Compare(binary.Left, binary.Right, scope, static (a, b) => Equals(a, b)), + PolicyBinaryOperator.NotEqual => Compare(binary.Left, binary.Right, scope, static (a, b) => !Equals(a, b)), + PolicyBinaryOperator.In => Contains(binary.Left, binary.Right, scope), + PolicyBinaryOperator.NotIn => new EvaluationValue(!Contains(binary.Left, binary.Right, scope).AsBoolean()), + PolicyBinaryOperator.LessThan => CompareNumeric(binary.Left, binary.Right, scope, static (a, b) => a < b), + PolicyBinaryOperator.LessThanOrEqual => CompareNumeric(binary.Left, binary.Right, scope, static (a, b) => a <= b), + PolicyBinaryOperator.GreaterThan => CompareNumeric(binary.Left, binary.Right, scope, static (a, b) => a > b), + PolicyBinaryOperator.GreaterThanOrEqual => CompareNumeric(binary.Left, binary.Right, scope, static (a, b) => a >= b), + _ => EvaluationValue.Null, + }; + } + + private EvaluationValue Compare(PolicyExpression left, PolicyExpression right, EvaluationScope scope, Func comparer) + { + var leftValue = Evaluate(left, scope).Raw; + var rightValue = Evaluate(right, scope).Raw; + return new EvaluationValue(comparer(leftValue, rightValue)); + } + + private EvaluationValue CompareNumeric(PolicyExpression left, PolicyExpression right, EvaluationScope scope, Func comparer) + { + var leftValue = Evaluate(left, scope).AsDecimal(); + var rightValue = Evaluate(right, scope).AsDecimal(); + return new EvaluationValue(leftValue.HasValue && rightValue.HasValue && comparer(leftValue.Value, rightValue.Value)); + } + + private EvaluationValue Contains(PolicyExpression needleExpr, PolicyExpression haystackExpr, EvaluationScope scope) + { + var needle = Evaluate(needleExpr, scope).Raw; + var haystack = Evaluate(haystackExpr, scope).Raw; + + if (haystack is ImmutableArray array) + { + return new EvaluationValue(array.Any(item => Equals(item, needle))); + } + + if (haystack is string str && needle is string needleString) + { + return new EvaluationValue(str.Contains(needleString, StringComparison.OrdinalIgnoreCase)); + } + + return new EvaluationValue(false); + } + + internal readonly struct EvaluationValue + { + public static readonly EvaluationValue Null = new(null); + public static readonly EvaluationValue True = new(true); + public static readonly EvaluationValue False = new(false); + + public EvaluationValue(object? raw) + { + Raw = raw; + } + + public object? Raw { get; } + + public bool AsBoolean() + { + return Raw switch + { + bool b => b, + string s => !string.IsNullOrWhiteSpace(s), + ImmutableArray array => !array.IsDefaultOrEmpty, + null => false, + _ => true, + }; + } + + public string? AsString() + { + return Raw switch + { + null => null, + string s => s, + decimal dec => dec.ToString("G", CultureInfo.InvariantCulture), + double d => d.ToString("G", CultureInfo.InvariantCulture), + int i => i.ToString(CultureInfo.InvariantCulture), + _ => Raw.ToString(), + }; + } + + public decimal? AsDecimal() + { + return Raw switch + { + decimal dec => dec, + double dbl => (decimal)dbl, + float fl => (decimal)fl, + int i => i, + long l => l, + string s when decimal.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var value) => value, + _ => null, + }; + } + } + + internal sealed class EvaluationScope + { + private readonly IReadOnlyDictionary locals; + + private EvaluationScope(IReadOnlyDictionary locals, PolicyEvaluationContext globals) + { + this.locals = locals; + Globals = globals; + } + + public static EvaluationScope Root(PolicyEvaluationContext globals) => + new EvaluationScope(new Dictionary(StringComparer.OrdinalIgnoreCase), globals); + + public static EvaluationScope FromLocals(PolicyEvaluationContext globals, IReadOnlyDictionary locals) => + new EvaluationScope(locals, globals); + + public bool TryGetLocal(string name, out object? value) + { + if (locals.TryGetValue(name, out value)) + { + return true; + } + + value = null; + return false; + } + + public PolicyEvaluationContext Globals { get; } + } + + private sealed class SeverityScope + { + private readonly PolicyEvaluationSeverity severity; + + public SeverityScope(PolicyEvaluationSeverity severity) + { + this.severity = severity; + } + + public EvaluationValue Get(string member) => member switch + { + "normalized" => new EvaluationValue(severity.Normalized), + "score" => new EvaluationValue(severity.Score), + _ => EvaluationValue.Null, + }; + } + + private sealed class EnvironmentScope + { + private readonly PolicyEvaluationEnvironment environment; + + public EnvironmentScope(PolicyEvaluationEnvironment environment) + { + this.environment = environment; + } + + public EvaluationValue Get(string member) + { + var value = environment.Get(member) + ?? environment.Get(member.ToLowerInvariant()); + return new EvaluationValue(value); + } + } + + private sealed class AdvisoryScope + { + private readonly PolicyEvaluationAdvisory advisory; + + public AdvisoryScope(PolicyEvaluationAdvisory advisory) + { + this.advisory = advisory; + } + + public EvaluationValue Get(string member) => member switch + { + "source" => new EvaluationValue(advisory.Source), + _ => advisory.Metadata.TryGetValue(member, out var value) ? new EvaluationValue(value) : EvaluationValue.Null, + }; + + public EvaluationValue Invoke(string member, ImmutableArray arguments, EvaluationScope scope, PolicyExpressionEvaluator evaluator) + { + if (member.Equals("has_metadata", StringComparison.OrdinalIgnoreCase)) + { + var key = arguments.Length > 0 ? evaluator.Evaluate(arguments[0], scope).AsString() : null; + if (string.IsNullOrEmpty(key)) + { + return EvaluationValue.False; + } + + return new EvaluationValue(advisory.Metadata.ContainsKey(key!)); + } + + return EvaluationValue.Null; + } + } + + private sealed class SbomScope + { + private readonly PolicyEvaluationSbom sbom; + + public SbomScope(PolicyEvaluationSbom sbom) + { + this.sbom = sbom; + } + + public EvaluationValue Get(string member) + { + if (member.Equals("tags", StringComparison.OrdinalIgnoreCase)) + { + return new EvaluationValue(sbom.Tags.ToImmutableArray()); + } + + return EvaluationValue.Null; + } + + public EvaluationValue HasTag(ImmutableArray arguments, EvaluationScope scope, PolicyExpressionEvaluator evaluator) + { + var tag = arguments.Length > 0 ? evaluator.Evaluate(arguments[0], scope).AsString() : null; + if (string.IsNullOrWhiteSpace(tag)) + { + return EvaluationValue.False; + } + + return new EvaluationValue(sbom.HasTag(tag!)); + } + } + + private sealed class VexScope + { + private readonly PolicyExpressionEvaluator evaluator; + private readonly PolicyEvaluationVexEvidence vex; + + public VexScope(PolicyExpressionEvaluator evaluator, PolicyEvaluationVexEvidence vex) + { + this.evaluator = evaluator; + this.vex = vex; + } + + public EvaluationValue Get(string member) => member switch + { + "status" => new EvaluationValue(vex.Statements.IsDefaultOrEmpty ? null : vex.Statements[0].Status), + "justification" => new EvaluationValue(vex.Statements.IsDefaultOrEmpty ? null : vex.Statements[0].Justification), + _ => EvaluationValue.Null, + }; + + public bool Any(ImmutableArray arguments, EvaluationScope scope) + { + if (arguments.Length == 0 || vex.Statements.IsDefaultOrEmpty) + { + return false; + } + + var predicate = arguments[0]; + foreach (var statement in vex.Statements) + { + var locals = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["status"] = statement.Status, + ["justification"] = statement.Justification, + ["statement"] = statement, + ["statementId"] = statement.StatementId, + }; + + var nestedScope = EvaluationScope.FromLocals(scope.Globals, locals); + if (evaluator.EvaluateBoolean(predicate, nestedScope)) + { + return true; + } + } + + return false; + } + + public PolicyEvaluationVexStatement? Latest() + { + if (vex.Statements.IsDefaultOrEmpty) + { + return null; + } + + return vex.Statements[^1]; + } + } +} diff --git a/src/StellaOps.Policy.Engine/Hosting/PolicyEngineStartupDiagnostics.cs b/src/StellaOps.Policy.Engine/Hosting/PolicyEngineStartupDiagnostics.cs new file mode 100644 index 00000000..4d7ccb23 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Hosting/PolicyEngineStartupDiagnostics.cs @@ -0,0 +1,12 @@ +using System.Threading; + +namespace StellaOps.Policy.Engine.Hosting; + +internal sealed class PolicyEngineStartupDiagnostics +{ + private int isReady; + + public bool IsReady => Volatile.Read(ref isReady) == 1; + + public void MarkReady() => Volatile.Write(ref isReady, 1); +} diff --git a/src/StellaOps.Policy.Engine/Options/PolicyEngineOptions.cs b/src/StellaOps.Policy.Engine/Options/PolicyEngineOptions.cs new file mode 100644 index 00000000..0b86996d --- /dev/null +++ b/src/StellaOps.Policy.Engine/Options/PolicyEngineOptions.cs @@ -0,0 +1,168 @@ +using System.Collections.ObjectModel; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Policy.Engine.Options; + +/// +/// Root configuration for the Policy Engine host. +/// +public sealed class PolicyEngineOptions +{ + public const string SectionName = "PolicyEngine"; + + public PolicyEngineAuthorityOptions Authority { get; } = new(); + + public PolicyEngineStorageOptions Storage { get; } = new(); + + public PolicyEngineWorkerOptions Workers { get; } = new(); + + public PolicyEngineResourceServerOptions ResourceServer { get; } = new(); + + public void Validate() + { + Authority.Validate(); + Storage.Validate(); + Workers.Validate(); + ResourceServer.Validate(); + } +} + +public sealed class PolicyEngineAuthorityOptions +{ + public bool Enabled { get; set; } = true; + + public string Issuer { get; set; } = "https://authority.stella-ops.local"; + + public string ClientId { get; set; } = "policy-engine"; + + public string? ClientSecret { get; set; } + + public IList Scopes { get; } = new List + { + StellaOpsScopes.PolicyRun, + StellaOpsScopes.FindingsRead, + StellaOpsScopes.EffectiveWrite + }; + + public int BackchannelTimeoutSeconds { get; set; } = 30; + + public void Validate() + { + if (!Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(Issuer)) + { + throw new InvalidOperationException("Policy Engine authority configuration requires an issuer."); + } + + if (!Uri.TryCreate(Issuer, UriKind.Absolute, out var issuerUri) || !issuerUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Policy Engine authority issuer must be an absolute URI."); + } + + if (issuerUri.Scheme != Uri.UriSchemeHttps && !issuerUri.IsLoopback) + { + throw new InvalidOperationException("Policy Engine authority issuer must use HTTPS unless targeting loopback."); + } + + if (string.IsNullOrWhiteSpace(ClientId)) + { + throw new InvalidOperationException("Policy Engine authority configuration requires a clientId."); + } + + if (Scopes.Count == 0) + { + throw new InvalidOperationException("Policy Engine authority configuration requires at least one scope."); + } + + if (BackchannelTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Policy Engine authority backchannel timeout must be greater than zero."); + } + } +} + +public sealed class PolicyEngineStorageOptions +{ + public string ConnectionString { get; set; } = "mongodb://localhost:27017/policy-engine"; + + public string DatabaseName { get; set; } = "policy_engine"; + + public int CommandTimeoutSeconds { get; set; } = 30; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Policy Engine storage configuration requires a MongoDB connection string."); + } + + if (string.IsNullOrWhiteSpace(DatabaseName)) + { + throw new InvalidOperationException("Policy Engine storage configuration requires a database name."); + } + + if (CommandTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Policy Engine storage command timeout must be greater than zero."); + } + } + + public TimeSpan CommandTimeout => TimeSpan.FromSeconds(CommandTimeoutSeconds); +} + +public sealed class PolicyEngineWorkerOptions +{ + public int SchedulerIntervalSeconds { get; set; } = 15; + + public int MaxConcurrentEvaluations { get; set; } = 4; + + public void Validate() + { + if (SchedulerIntervalSeconds <= 0) + { + throw new InvalidOperationException("Policy Engine worker interval must be greater than zero."); + } + + if (MaxConcurrentEvaluations <= 0) + { + throw new InvalidOperationException("Policy Engine worker concurrency must be greater than zero."); + } + } +} + +public sealed class PolicyEngineResourceServerOptions +{ + public string Authority { get; set; } = "https://authority.stella-ops.local"; + + public IList Audiences { get; } = new List { "api://policy-engine" }; + + public IList RequiredScopes { get; } = new List { StellaOpsScopes.PolicyRun }; + + public IList RequiredTenants { get; } = new List(); + + public IList BypassNetworks { get; } = new List { "127.0.0.1/32", "::1/128" }; + + public bool RequireHttpsMetadata { get; set; } = true; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Authority)) + { + throw new InvalidOperationException("Resource server configuration requires an Authority URL."); + } + + if (!Uri.TryCreate(Authority.Trim(), UriKind.Absolute, out var uri)) + { + throw new InvalidOperationException("Resource server Authority URL must be absolute."); + } + + if (RequireHttpsMetadata && !uri.IsLoopback && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Resource server Authority URL must use HTTPS when HTTPS metadata is required."); + } + } +} diff --git a/src/StellaOps.Policy.Engine/Program.cs b/src/StellaOps.Policy.Engine/Program.cs index a891055c..842f78ab 100644 --- a/src/StellaOps.Policy.Engine/Program.cs +++ b/src/StellaOps.Policy.Engine/Program.cs @@ -1,19 +1,137 @@ +using System.IO; +using Microsoft.Extensions.Options; +using NetEscapades.Configuration.Yaml; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Configuration; +using StellaOps.Policy.Engine.Hosting; +using StellaOps.Policy.Engine.Options; +using StellaOps.Policy.Engine.Compilation; +using StellaOps.Policy.Engine.Endpoints; +using StellaOps.Policy.Engine.Services; +using StellaOps.Policy.Engine.Workers; + var builder = WebApplication.CreateBuilder(args); -builder.Configuration - .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables("POLICY_"); +builder.Logging.ClearProviders(); +builder.Logging.AddConsole(); -builder.Services.AddOptions(); -builder.Services.AddLogging(); +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "STELLAOPS_POLICY_ENGINE_"; + options.ConfigureBuilder = configurationBuilder => + { + var contentRoot = builder.Environment.ContentRootPath; + foreach (var relative in new[] + { + "../etc/policy-engine.yaml", + "../etc/policy-engine.local.yaml", + "policy-engine.yaml", + "policy-engine.local.yaml" + }) + { + var path = Path.Combine(contentRoot, relative); + configurationBuilder.AddYamlFile(path, optional: true); + } + }; +}); -// TODO: wire Authority client + Policy engine services once implemented tasks land. +var bootstrap = StellaOpsConfigurationBootstrapper.Build(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "STELLAOPS_POLICY_ENGINE_"; + options.BindingSection = PolicyEngineOptions.SectionName; + options.ConfigureBuilder = configurationBuilder => + { + foreach (var relative in new[] + { + "../etc/policy-engine.yaml", + "../etc/policy-engine.local.yaml", + "policy-engine.yaml", + "policy-engine.local.yaml" + }) + { + var path = Path.Combine(builder.Environment.ContentRootPath, relative); + configurationBuilder.AddYamlFile(path, optional: true); + } + }; + options.PostBind = static (value, _) => value.Validate(); +}); + +builder.Configuration.AddConfiguration(bootstrap.Configuration); + +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(PolicyEngineOptions.SectionName)) + .Validate(options => + { + try + { + options.Validate(); + return true; + } + catch (Exception ex) + { + throw new OptionsValidationException( + PolicyEngineOptions.SectionName, + typeof(PolicyEngineOptions), + new[] { ex.Message }); + } + }) + .ValidateOnStart(); + +builder.Services.AddSingleton(sp => sp.GetRequiredService>().Value); +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +builder.Services.AddHttpContextAccessor(); +builder.Services.AddRouting(options => options.LowercaseUrls = true); +builder.Services.AddProblemDetails(); +builder.Services.AddHealthChecks(); + +builder.Services.AddAuthentication(); +builder.Services.AddAuthorization(); +builder.Services.AddStellaOpsScopeHandler(); +builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: $"{PolicyEngineOptions.SectionName}:ResourceServer"); + +if (bootstrap.Options.Authority.Enabled) +{ + builder.Services.AddStellaOpsAuthClient(clientOptions => + { + clientOptions.Authority = bootstrap.Options.Authority.Issuer; + clientOptions.ClientId = bootstrap.Options.Authority.ClientId; + clientOptions.ClientSecret = bootstrap.Options.Authority.ClientSecret; + clientOptions.HttpTimeout = TimeSpan.FromSeconds(bootstrap.Options.Authority.BackchannelTimeoutSeconds); + + clientOptions.DefaultScopes.Clear(); + foreach (var scope in bootstrap.Options.Authority.Scopes) + { + clientOptions.DefaultScopes.Add(scope); + } + }); +} var app = builder.Build(); -app.MapGet("/healthz", () => Results.Ok(new { status = "ok" })); -app.MapGet("/readyz", () => Results.Ok(new { status = "warming" })); +app.UseAuthentication(); +app.UseAuthorization(); + +app.MapHealthChecks("/healthz"); +app.MapGet("/readyz", (PolicyEngineStartupDiagnostics diagnostics) => + diagnostics.IsReady + ? Results.Ok(new { status = "ready" }) + : Results.StatusCode(StatusCodes.Status503ServiceUnavailable)) + .WithName("Readiness"); app.MapGet("/", () => Results.Redirect("/healthz")); +app.MapPolicyCompilation(); + app.Run(); diff --git a/src/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs b/src/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..826700d8 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Policy.Engine.Tests")] diff --git a/src/StellaOps.Policy.Engine/README.md b/src/StellaOps.Policy.Engine/README.md new file mode 100644 index 00000000..2531805e --- /dev/null +++ b/src/StellaOps.Policy.Engine/README.md @@ -0,0 +1,14 @@ +# Policy Engine Host Template + +This service hosts the Policy Engine APIs and background workers introduced in **Policy Engine v2**. The project currently ships a minimal bootstrap that validates configuration, registers Authority clients, and exposes readiness/health endpoints. Future tasks will extend it with compilation, evaluation, and persistence features. + +## Compliance Checklist + +- [x] Configuration loads from `policy-engine.yaml`/environment variables and validates on startup. +- [x] Authority client scaffolding enforces `policy:*` + `effective:write` scopes and respects back-channel timeouts. +- [x] Resource server authentication requires Policy Engine scopes with tenant-aware policies. +- [x] Health and readiness endpoints exist for platform probes. +- [ ] Deterministic policy evaluation pipeline implemented (POLICY-ENGINE-20-002). +- [ ] Mongo materialisation writers implemented (POLICY-ENGINE-20-004). +- [ ] Observability (metrics/traces/logs) completed (POLICY-ENGINE-20-007). +- [ ] Comprehensive test suites and perf baselines established (POLICY-ENGINE-20-008). diff --git a/src/StellaOps.Policy.Engine/Services/PolicyCompilationService.cs b/src/StellaOps.Policy.Engine/Services/PolicyCompilationService.cs new file mode 100644 index 00000000..74dd6212 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Services/PolicyCompilationService.cs @@ -0,0 +1,120 @@ +using System.Collections.Immutable; +using StellaOps.Policy; +using StellaOps.Policy.Engine.Compilation; + +namespace StellaOps.Policy.Engine.Services; + +/// +/// Provides deterministic compilation for stella-dsl@1 policy documents and exposes +/// basic statistics consumed by API/CLI surfaces. +/// +internal sealed class PolicyCompilationService +{ + private readonly PolicyCompiler compiler; + + public PolicyCompilationService(PolicyCompiler compiler) + { + this.compiler = compiler ?? throw new ArgumentNullException(nameof(compiler)); + } + + public PolicyCompilationResultDto Compile(PolicyCompileRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (request.Dsl is null || string.IsNullOrWhiteSpace(request.Dsl.Source)) + { + throw new ArgumentException("Compilation requires DSL source.", nameof(request)); + } + + if (!string.Equals(request.Dsl.Syntax, "stella-dsl@1", StringComparison.Ordinal)) + { + return PolicyCompilationResultDto.FromFailure( + ImmutableArray.Create(PolicyIssue.Error( + PolicyDslDiagnosticCodes.UnsupportedSyntaxVersion, + $"Unsupported syntax '{request.Dsl.Syntax ?? "null"}'. Expected 'stella-dsl@1'.", + "dsl.syntax"))); + } + + var result = compiler.Compile(request.Dsl.Source); + if (!result.Success || result.Document is null) + { + return PolicyCompilationResultDto.FromFailure(result.Diagnostics); + } + + return PolicyCompilationResultDto.FromSuccess(result); + } +} + +internal sealed record PolicyCompileRequest(PolicyDslPayload Dsl); + +internal sealed record PolicyDslPayload(string Syntax, string Source); + +internal sealed record PolicyCompilationResultDto( + bool Success, + string? Digest, + PolicyCompilationStatistics? Statistics, + ImmutableArray Diagnostics) +{ + public static PolicyCompilationResultDto FromFailure(ImmutableArray diagnostics) => + new(false, null, null, diagnostics); + + public static PolicyCompilationResultDto FromSuccess(PolicyCompilationResult compilationResult) + { + if (compilationResult.Document is null) + { + throw new ArgumentException("Compilation result must include a document for success.", nameof(compilationResult)); + } + + var stats = PolicyCompilationStatistics.Create(compilationResult.Document); + return new PolicyCompilationResultDto( + true, + $"sha256:{compilationResult.Checksum}", + stats, + compilationResult.Diagnostics); + } +} + +internal sealed record PolicyCompilationStatistics( + int RuleCount, + ImmutableDictionary ActionCounts) +{ + public static PolicyCompilationStatistics Create(PolicyIrDocument document) + { + var actions = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + + void Increment(string key) + { + actions[key] = actions.TryGetValue(key, out var existing) ? existing + 1 : 1; + } + + foreach (var rule in document.Rules) + { + foreach (var action in rule.ThenActions) + { + Increment(GetActionKey(action)); + } + + foreach (var action in rule.ElseActions) + { + Increment($"else:{GetActionKey(action)}"); + } + } + + return new PolicyCompilationStatistics(document.Rules.Length, actions.ToImmutable()); + } + + private static string GetActionKey(PolicyIrAction action) => action switch + { + PolicyIrAssignmentAction => "assign", + PolicyIrAnnotateAction => "annotate", + PolicyIrIgnoreAction => "ignore", + PolicyIrEscalateAction => "escalate", + PolicyIrRequireVexAction => "requireVex", + PolicyIrWarnAction => "warn", + PolicyIrDeferAction => "defer", + _ => "unknown" + }; +} diff --git a/src/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs b/src/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs new file mode 100644 index 00000000..c09f27d9 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Services/PolicyEvaluationService.cs @@ -0,0 +1,26 @@ +using System.Collections.Immutable; +using StellaOps.Policy.Engine.Compilation; +using StellaOps.Policy.Engine.Evaluation; + +namespace StellaOps.Policy.Engine.Services; + +internal sealed class PolicyEvaluationService +{ + private readonly PolicyEvaluator evaluator = new(); + + public PolicyEvaluationResult Evaluate(PolicyIrDocument document, PolicyEvaluationContext context) + { + if (document is null) + { + throw new ArgumentNullException(nameof(document)); + } + + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + var request = new PolicyEvaluationRequest(document, context); + return evaluator.Evaluate(request); + } +} diff --git a/src/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj b/src/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj index 73c8b7e3..a33dbe6f 100644 --- a/src/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj +++ b/src/StellaOps.Policy.Engine/StellaOps.Policy.Engine.csproj @@ -12,5 +12,8 @@ + + + diff --git a/src/StellaOps.Policy.Engine/TASKS.md b/src/StellaOps.Policy.Engine/TASKS.md index 95c85ce0..dec3f04d 100644 --- a/src/StellaOps.Policy.Engine/TASKS.md +++ b/src/StellaOps.Policy.Engine/TASKS.md @@ -2,10 +2,13 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| POLICY-ENGINE-20-000 | TODO | Policy Guild, BE-Base Platform Guild | POLICY-AOC-19-001 | Spin up new `StellaOps.Policy.Engine` service project (minimal API host + worker), wire DI composition root, configuration binding, and Authority client scaffolding. | New project builds/tests; registered in solution; bootstrap validates configuration; host template committed with compliance checklist. | -| POLICY-ENGINE-20-001 | TODO | Policy Guild, Language Infrastructure Guild | POLICY-ENGINE-20-000 | Implement `stella-dsl@1` parser + IR compiler with grammar validation, syntax diagnostics, and checksum outputs for caching. | DSL parser handles full grammar + error reporting; IR checksum stored with policy version; unit tests cover success/error paths. | -| POLICY-ENGINE-20-002 | TODO | Policy Guild | POLICY-ENGINE-20-001 | Build deterministic evaluator honoring lexical/priority order, first-match semantics, and safe value types (no wall-clock/network access). | Evaluator executes policies deterministically in unit/property tests; guard rejects forbidden intrinsics; perf baseline recorded. | +| POLICY-ENGINE-20-000 | DONE (2025-10-26) | Policy Guild, BE-Base Platform Guild | POLICY-AOC-19-001 | Spin up new `StellaOps.Policy.Engine` service project (minimal API host + worker), wire DI composition root, configuration binding, and Authority client scaffolding. | New project builds/tests; registered in solution; bootstrap validates configuration; host template committed with compliance checklist. | +> 2025-10-26: Added policy-engine host bootstrap (config, auth client, resource server auth, readiness probe) + sample YAML and compliance readme. +| POLICY-ENGINE-20-001 | DONE (2025-10-26) | Policy Guild, Language Infrastructure Guild | POLICY-ENGINE-20-000 | Implement `stella-dsl@1` parser + IR compiler with grammar validation, syntax diagnostics, and checksum outputs for caching. | DSL parser handles full grammar + error reporting; IR checksum stored with policy version; unit tests cover success/error paths. | +| POLICY-ENGINE-20-002 | BLOCKED (2025-10-26) | Policy Guild | POLICY-ENGINE-20-001 | Build deterministic evaluator honoring lexical/priority order, first-match semantics, and safe value types (no wall-clock/network access). | Evaluator executes policies deterministically in unit/property tests; guard rejects forbidden intrinsics; perf baseline recorded. | +> 2025-10-26: Blocked while bootstrapping DSL parser/evaluator; remaining grammar coverage (profile keywords, condition parsing) and rule evaluation semantics still pending to satisfy acceptance tests. | POLICY-ENGINE-20-003 | TODO | Policy Guild, Concelier Core Guild, Excititor Core Guild | POLICY-ENGINE-20-001, CONCELIER-POLICY-20-002, EXCITITOR-POLICY-20-002 | Implement selection joiners resolving SBOM↔advisory↔VEX tuples using linksets and PURL equivalence tables, with deterministic batching. | Joiners fetch correct candidate sets in integration tests; batching meets memory targets; explain traces list input provenance. | +> 2025-10-26: Scheduler DTO contracts for runs/diffs/explains available (`src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md`); consume `PolicyRunRequest/Status/DiffSummary` from samples under `samples/api/scheduler/`. | POLICY-ENGINE-20-004 | TODO | Policy Guild, Platform Storage Guild | POLICY-ENGINE-20-003, CONCELIER-POLICY-20-003, EXCITITOR-POLICY-20-003 | Ship materialization writer that upserts into `effective_finding_{policyId}` with append-only history, tenant scoping, and trace references. | Writes restricted to Policy Engine identity; idempotent upserts proven via tests; collections indexed per design and docs updated. | | POLICY-ENGINE-20-005 | TODO | Policy Guild, Security Engineering | POLICY-ENGINE-20-002 | Enforce determinism guard banning wall-clock, RNG, and network usage during evaluation via static analysis + runtime sandbox. | Guard blocks forbidden APIs in unit/integration tests; violations emit `ERR_POL_004`; CI analyzer wired. | | POLICY-ENGINE-20-006 | TODO | Policy Guild, Scheduler Worker Guild | POLICY-ENGINE-20-003, POLICY-ENGINE-20-004, SCHED-WORKER-20-301 | Implement incremental orchestrator reacting to advisory/vex/SBOM change streams and scheduling partial policy re-evaluations. | Change stream listeners enqueue affected tuples with dedupe; orchestrator meets 5 min SLA in perf tests; metrics exposed (`policy_run_seconds`). | @@ -37,6 +40,7 @@ | POLICY-ENGINE-30-001 | TODO | Policy Guild, Cartographer Guild | POLICY-ENGINE-20-004, CARTO-GRAPH-21-005 | Define overlay contract for graph nodes/edges (status, severity, rationale refs, path relevance), expose projection API for Cartographer, and document schema versioning. | Overlay contract published (OpenAPI + schema); integration tests validate payloads against fixtures; versioning strategy documented. | | POLICY-ENGINE-30-002 | TODO | Policy Guild, Cartographer Guild | POLICY-ENGINE-30-001, CARTO-GRAPH-21-006 | Implement simulation bridge returning on-the-fly overlays for Cartographer/Graph Explorer when invoking Policy Engine simulate; ensure no writes and deterministic outputs. | Simulation API returns overlays within SLA; end-to-end test from Graph Explorer consumes results; docs updated. | | POLICY-ENGINE-30-003 | TODO | Policy Guild, Scheduler Guild, Cartographer Guild | POLICY-ENGINE-20-006, CARTO-GRAPH-21-007 | Emit change events (`policy.effective.updated`) with graph-friendly payloads so Cartographer overlay worker refreshes nodes/edges within 2 minutes. | Event published on run completion; Cartographer listener integration test passes; metrics capture lag. | +| POLICY-ENGINE-30-101 | TODO | Policy Guild | POLICY-ENGINE-29-001 | Surface trust weighting configuration (issuer base weights, signature modifiers, recency decay, scope adjustments) for VEX Lens via Policy Studio + API; ensure deterministic evaluation. | Trust weighting config exposed; Policy Studio UI updated; integration tests verify VEX Lens consumption. | ## Link-Not-Merge v1 diff --git a/src/StellaOps.Policy.Engine/Workers/PolicyEngineBootstrapWorker.cs b/src/StellaOps.Policy.Engine/Workers/PolicyEngineBootstrapWorker.cs new file mode 100644 index 00000000..dbb57d17 --- /dev/null +++ b/src/StellaOps.Policy.Engine/Workers/PolicyEngineBootstrapWorker.cs @@ -0,0 +1,35 @@ +using System; +using System.Threading; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StellaOps.Policy.Engine.Hosting; +using StellaOps.Policy.Engine.Options; + +namespace StellaOps.Policy.Engine.Workers; + +internal sealed class PolicyEngineBootstrapWorker : BackgroundService +{ + private readonly ILogger logger; + private readonly PolicyEngineStartupDiagnostics diagnostics; + private readonly PolicyEngineOptions options; + + public PolicyEngineBootstrapWorker( + ILogger logger, + PolicyEngineStartupDiagnostics diagnostics, + PolicyEngineOptions options) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + this.options = options ?? throw new ArgumentNullException(nameof(options)); + } + + protected override Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation("Policy Engine bootstrap worker started. Authority issuer: {AuthorityIssuer}. Database: {Database}.", + options.Authority.Issuer, + options.Storage.DatabaseName); + + diagnostics.MarkReady(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Policy/PolicyDocument.cs b/src/StellaOps.Policy/PolicyDocument.cs index a086109d..a70ca0ce 100644 --- a/src/StellaOps.Policy/PolicyDocument.cs +++ b/src/StellaOps.Policy/PolicyDocument.cs @@ -30,10 +30,12 @@ public static class PolicySchema } var lower = fileName.Trim().ToLowerInvariant(); - if (lower.EndsWith(".yaml", StringComparison.Ordinal) || lower.EndsWith(".yml", StringComparison.Ordinal)) - { - return PolicyDocumentFormat.Yaml; - } + if (lower.EndsWith(".yaml", StringComparison.Ordinal) + || lower.EndsWith(".yml", StringComparison.Ordinal) + || lower.EndsWith(".stella", StringComparison.Ordinal)) + { + return PolicyDocumentFormat.Yaml; + } return PolicyDocumentFormat.Json; } diff --git a/src/StellaOps.Policy/PolicyValidationCli.cs b/src/StellaOps.Policy/PolicyValidationCli.cs index 98884988..51b764c9 100644 --- a/src/StellaOps.Policy/PolicyValidationCli.cs +++ b/src/StellaOps.Policy/PolicyValidationCli.cs @@ -215,9 +215,10 @@ public sealed class PolicyValidationCli private static bool MatchesPolicyExtension(string path) { var extension = Path.GetExtension(path); - return extension.Equals(".yaml", StringComparison.OrdinalIgnoreCase) - || extension.Equals(".yml", StringComparison.OrdinalIgnoreCase) - || extension.Equals(".json", StringComparison.OrdinalIgnoreCase); + return extension.Equals(".yaml", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".yml", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".json", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".stella", StringComparison.OrdinalIgnoreCase); } private static string MakeRelative(string path) diff --git a/src/StellaOps.Policy/StellaOps.Policy.csproj b/src/StellaOps.Policy/StellaOps.Policy.csproj index 4b97757c..b1d74ac2 100644 --- a/src/StellaOps.Policy/StellaOps.Policy.csproj +++ b/src/StellaOps.Policy/StellaOps.Policy.csproj @@ -1,22 +1,22 @@ - - - net10.0 - enable - enable - preview - true - - - - - - - - - - - - - - - + + + net10.0 + enable + enable + preview + true + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Registry.TokenService.Tests/PlanRegistryTests.cs b/src/StellaOps.Registry.TokenService.Tests/PlanRegistryTests.cs new file mode 100644 index 00000000..e3deccc3 --- /dev/null +++ b/src/StellaOps.Registry.TokenService.Tests/PlanRegistryTests.cs @@ -0,0 +1,109 @@ +using System.Security.Claims; +using Microsoft.Extensions.Options; +using StellaOps.Registry.TokenService; + +namespace StellaOps.Registry.TokenService.Tests; + +public sealed class PlanRegistryTests +{ + private static RegistryTokenServiceOptions CreateOptions() + { + return new RegistryTokenServiceOptions + { + Authority = new RegistryTokenServiceOptions.AuthorityOptions + { + Issuer = "https://authority.localhost", + RequireHttpsMetadata = false, + }, + Signing = new RegistryTokenServiceOptions.SigningOptions + { + Issuer = "https://registry.localhost/token", + KeyPath = Path.GetTempFileName(), + }, + Registry = new RegistryTokenServiceOptions.RegistryOptions + { + Realm = "https://registry.localhost/v2/token" + }, + Plans = + { + new RegistryTokenServiceOptions.PlanRule + { + Name = "community", + Repositories = + { + new RegistryTokenServiceOptions.RepositoryRule + { + Pattern = "stella-ops/public/*", + Actions = new [] { "pull" } + } + } + }, + new RegistryTokenServiceOptions.PlanRule + { + Name = "enterprise", + Repositories = + { + new RegistryTokenServiceOptions.RepositoryRule + { + Pattern = "stella-ops/public/*", + Actions = new [] { "pull" } + }, + new RegistryTokenServiceOptions.RepositoryRule + { + Pattern = "stella-ops/enterprise/*", + Actions = new [] { "pull", "push" } + } + } + } + } + }; + } + + [Fact] + public void Authorize_AllowsMatchingPlan() + { + var options = CreateOptions(); + options.Signing.Validate(); + options.Registry.Validate(); + foreach (var plan in options.Plans) + { + plan.Validate(); + } + + var registry = new PlanRegistry(options); + + var principal = new ClaimsPrincipal(new ClaimsIdentity(new[] + { + new Claim("stellaops:plan", "enterprise") + }, "test")); + + var decision = registry.Authorize(principal, new[] + { + new RegistryAccessRequest("repository", "stella-ops/enterprise/cache", new [] { "pull" }) + }); + + Assert.True(decision.Allowed); + } + + [Fact] + public void Authorize_DeniesUnknownPlan() + { + var options = CreateOptions(); + options.Signing.Validate(); + options.Registry.Validate(); + foreach (var plan in options.Plans) + { + plan.Validate(); + } + + var registry = new PlanRegistry(options); + var principal = new ClaimsPrincipal(new ClaimsIdentity(new Claim[] { }, "test")); + + var decision = registry.Authorize(principal, new[] + { + new RegistryAccessRequest("repository", "stella-ops/enterprise/cache", new [] { "pull" }) + }); + + Assert.False(decision.Allowed); + } +} diff --git a/src/StellaOps.Registry.TokenService.Tests/RegistryScopeParserTests.cs b/src/StellaOps.Registry.TokenService.Tests/RegistryScopeParserTests.cs new file mode 100644 index 00000000..01dee736 --- /dev/null +++ b/src/StellaOps.Registry.TokenService.Tests/RegistryScopeParserTests.cs @@ -0,0 +1,38 @@ +using Microsoft.AspNetCore.Http; +using StellaOps.Registry.TokenService; + +namespace StellaOps.Registry.TokenService.Tests; + +public sealed class RegistryScopeParserTests +{ + [Fact] + public void Parse_SingleScope_DefaultsPull() + { + var query = new QueryCollection(new Dictionary + { + ["scope"] = "repository:stella-ops/public/base" + }); + + var result = RegistryScopeParser.Parse(query); + + Assert.Single(result); + Assert.Equal("repository", result[0].Type); + Assert.Equal("stella-ops/public/base", result[0].Name); + Assert.Equal(new[] { "pull" }, result[0].Actions); + } + + [Fact] + public void Parse_MultipleScopes() + { + var query = new QueryCollection(new Dictionary + { + ["scope"] = new[] { "repository:stella/public/api:pull,push", "repository:stella/private/api:pull" } + }); + + var result = RegistryScopeParser.Parse(query); + + Assert.Equal(2, result.Count); + Assert.Equal(new[] { "pull", "push" }, result[0].Actions); + Assert.Equal(new[] { "pull" }, result[1].Actions); + } +} diff --git a/src/StellaOps.Registry.TokenService.Tests/RegistryTokenIssuerTests.cs b/src/StellaOps.Registry.TokenService.Tests/RegistryTokenIssuerTests.cs new file mode 100644 index 00000000..d7d8f59c --- /dev/null +++ b/src/StellaOps.Registry.TokenService.Tests/RegistryTokenIssuerTests.cs @@ -0,0 +1,110 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Security.Claims; +using System.Security.Cryptography; +using Microsoft.Extensions.Options; +using StellaOps.Registry.TokenService; +using StellaOps.Registry.TokenService.Observability; + +namespace StellaOps.Registry.TokenService.Tests; + +public sealed class RegistryTokenIssuerTests : IDisposable +{ + private readonly List _tempFiles = new(); + + [Fact] + public void IssueToken_GeneratesJwtWithAccessClaim() + { + var pemPath = CreatePemKey(); + var options = new RegistryTokenServiceOptions + { + Authority = new RegistryTokenServiceOptions.AuthorityOptions + { + Issuer = "https://authority.localhost", + RequireHttpsMetadata = false, + }, + Signing = new RegistryTokenServiceOptions.SigningOptions + { + Issuer = "https://registry.localhost/token", + KeyPath = pemPath, + Lifetime = TimeSpan.FromMinutes(5) + }, + Registry = new RegistryTokenServiceOptions.RegistryOptions + { + Realm = "https://registry.localhost/v2/token" + }, + Plans = + { + new RegistryTokenServiceOptions.PlanRule + { + Name = "community", + Repositories = + { + new RegistryTokenServiceOptions.RepositoryRule + { + Pattern = "stella-ops/public/*", + Actions = new [] { "pull" } + } + } + } + } + }; + options.Validate(); + + var issuer = new RegistryTokenIssuer( + Options.Create(options), + new PlanRegistry(options), + new RegistryTokenMetrics(), + TimeProvider.System); + + var principal = new ClaimsPrincipal(new ClaimsIdentity(new[] + { + new Claim("sub", "client-1"), + new Claim("stellaops:plan", "community") + }, "test")); + + var accessRequests = new[] + { + new RegistryAccessRequest("repository", "stella-ops/public/base", new [] { "pull" }) + }; + + var response = issuer.IssueToken(principal, "registry.localhost", accessRequests); + + Assert.NotEmpty(response.Token); + + var handler = new JwtSecurityTokenHandler(); + var jwt = handler.ReadJwtToken(response.Token); + + Assert.Equal("https://registry.localhost/token", jwt.Issuer); + Assert.True(jwt.Payload.TryGetValue("access", out var access)); + Assert.NotNull(access); + } + + private string CreatePemKey() + { + using var rsa = RSA.Create(2048); + var builder = new StringWriter(); + builder.WriteLine("-----BEGIN PRIVATE KEY-----"); + builder.WriteLine(Convert.ToBase64String(rsa.ExportPkcs8PrivateKey(), Base64FormattingOptions.InsertLineBreaks)); + builder.WriteLine("-----END PRIVATE KEY-----"); + + var path = Path.GetTempFileName(); + File.WriteAllText(path, builder.ToString()); + _tempFiles.Add(path); + return path; + } + + public void Dispose() + { + foreach (var file in _tempFiles) + { + try + { + File.Delete(file); + } + catch + { + // ignore + } + } + } +} diff --git a/src/StellaOps.Registry.TokenService.Tests/StellaOps.Registry.TokenService.Tests.csproj b/src/StellaOps.Registry.TokenService.Tests/StellaOps.Registry.TokenService.Tests.csproj new file mode 100644 index 00000000..efa1033b --- /dev/null +++ b/src/StellaOps.Registry.TokenService.Tests/StellaOps.Registry.TokenService.Tests.csproj @@ -0,0 +1,28 @@ + + + + net10.0 + enable + enable + Exe + false + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Registry.TokenService.Tests/UnitTest1.cs b/src/StellaOps.Registry.TokenService.Tests/UnitTest1.cs new file mode 100644 index 00000000..5760479c --- /dev/null +++ b/src/StellaOps.Registry.TokenService.Tests/UnitTest1.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Registry.TokenService.Tests; + +public class UnitTest1 +{ + [Fact] + public void Test1() + { + + } +} diff --git a/src/StellaOps.Registry.TokenService.Tests/xunit.runner.json b/src/StellaOps.Registry.TokenService.Tests/xunit.runner.json new file mode 100644 index 00000000..86c7ea05 --- /dev/null +++ b/src/StellaOps.Registry.TokenService.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/src/StellaOps.Registry.TokenService/Observability/RegistryTokenMetrics.cs b/src/StellaOps.Registry.TokenService/Observability/RegistryTokenMetrics.cs new file mode 100644 index 00000000..22452cf4 --- /dev/null +++ b/src/StellaOps.Registry.TokenService/Observability/RegistryTokenMetrics.cs @@ -0,0 +1,34 @@ +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Registry.TokenService.Observability; + +public sealed class RegistryTokenMetrics : IDisposable +{ + public const string MeterName = "StellaOps.Registry.TokenService"; + + private readonly Meter _meter; + private bool _disposed; + + public RegistryTokenMetrics() + { + _meter = new Meter(MeterName); + TokensIssued = _meter.CreateCounter("registry_token_issued_total", unit: "tokens", description: "Total tokens issued grouped by plan."); + TokensRejected = _meter.CreateCounter("registry_token_rejected_total", unit: "tokens", description: "Total token requests rejected grouped by reason."); + } + + public Counter TokensIssued { get; } + + public Counter TokensRejected { get; } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _meter.Dispose(); + _disposed = true; + } +} diff --git a/src/StellaOps.Registry.TokenService/PlanRegistry.cs b/src/StellaOps.Registry.TokenService/PlanRegistry.cs new file mode 100644 index 00000000..a5f3a6ed --- /dev/null +++ b/src/StellaOps.Registry.TokenService/PlanRegistry.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Text.RegularExpressions; + +namespace StellaOps.Registry.TokenService; + +/// +/// Evaluates repository access against configured plan rules. +/// +public sealed class PlanRegistry +{ + private readonly IReadOnlyDictionary _plans; + private readonly IReadOnlySet _revokedLicenses; + private readonly string? _defaultPlan; + + public PlanRegistry(RegistryTokenServiceOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + _plans = options.Plans + .Select(plan => new PlanDescriptor(plan)) + .ToDictionary(static plan => plan.Name, StringComparer.OrdinalIgnoreCase); + + _revokedLicenses = options.RevokedLicenses.Count == 0 + ? new HashSet(StringComparer.OrdinalIgnoreCase) + : new HashSet(options.RevokedLicenses, StringComparer.OrdinalIgnoreCase); + + _defaultPlan = options.DefaultPlan; + } + + public RegistryAccessDecision Authorize( + ClaimsPrincipal principal, + IReadOnlyList requests) + { + ArgumentNullException.ThrowIfNull(principal); + ArgumentNullException.ThrowIfNull(requests); + + if (requests.Count == 0) + { + return new RegistryAccessDecision(false, "no_scopes_requested"); + } + + var licenseId = principal.FindFirstValue("stellaops:license")?.Trim(); + if (!string.IsNullOrEmpty(licenseId) && _revokedLicenses.Contains(licenseId)) + { + return new RegistryAccessDecision(false, "license_revoked"); + } + + var planName = principal.FindFirstValue("stellaops:plan")?.Trim(); + if (string.IsNullOrEmpty(planName)) + { + planName = _defaultPlan; + } + + if (string.IsNullOrEmpty(planName) || !_plans.TryGetValue(planName, out var descriptor)) + { + return new RegistryAccessDecision(false, "plan_unknown"); + } + + foreach (var request in requests) + { + if (!descriptor.IsRepositoryAllowed(request)) + { + return new RegistryAccessDecision(false, "scope_not_permitted"); + } + } + + return new RegistryAccessDecision(true); + } + + private sealed class PlanDescriptor + { + private readonly IReadOnlyList _repositories; + + public PlanDescriptor(RegistryTokenServiceOptions.PlanRule source) + { + Name = source.Name; + _repositories = source.Repositories + .Select(rule => new RepositoryDescriptor(rule)) + .ToArray(); + } + + public string Name { get; } + + public bool IsRepositoryAllowed(RegistryAccessRequest request) + { + if (!string.Equals(request.Type, "repository", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + foreach (var repo in _repositories) + { + if (!repo.Matches(request.Name)) + { + continue; + } + + if (repo.AllowsActions(request.Actions)) + { + return true; + } + } + + return false; + } + } + + private sealed class RepositoryDescriptor + { + private readonly Regex _pattern; + private readonly IReadOnlySet _allowedActions; + + public RepositoryDescriptor(RegistryTokenServiceOptions.RepositoryRule rule) + { + Pattern = rule.Pattern; + _pattern = Compile(rule.Pattern); + _allowedActions = new HashSet(rule.Actions, StringComparer.OrdinalIgnoreCase); + } + + public string Pattern { get; } + + public bool Matches(string repository) + { + return _pattern.IsMatch(repository); + } + + public bool AllowsActions(IReadOnlyList actions) + { + foreach (var action in actions) + { + if (!_allowedActions.Contains(action)) + { + return false; + } + } + + return true; + } + + private static Regex Compile(string pattern) + { + var escaped = Regex.Escape(pattern); + escaped = escaped.Replace(@"\*", ".*", StringComparison.Ordinal); + return new Regex($"^{escaped}$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); + } + } +} diff --git a/src/StellaOps.Registry.TokenService/Program.cs b/src/StellaOps.Registry.TokenService/Program.cs new file mode 100644 index 00000000..52dcdc4e --- /dev/null +++ b/src/StellaOps.Registry.TokenService/Program.cs @@ -0,0 +1,171 @@ +using System.Net; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using OpenTelemetry.Instrumentation.AspNetCore; +using OpenTelemetry.Instrumentation.Runtime; +using OpenTelemetry.Metrics; +using Serilog; +using Serilog.Events; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Configuration; +using StellaOps.Registry.TokenService; +using StellaOps.Registry.TokenService.Observability; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "REGISTRY_TOKEN_"; + options.ConfigureBuilder = configurationBuilder => + { + configurationBuilder.AddYamlFile("../etc/registry-token.yaml", optional: true, reloadOnChange: true); + }; +}); + +var bootstrapOptions = builder.Configuration.BindOptions( + RegistryTokenServiceOptions.SectionName, + (opts, _) => opts.Validate()); + +builder.Host.UseSerilog((context, services, loggerConfiguration) => +{ + loggerConfiguration + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) + .Enrich.FromLogContext() + .WriteTo.Console(); +}); + +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(RegistryTokenServiceOptions.SectionName)) + .PostConfigure(options => options.Validate()) + .ValidateOnStart(); + +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => +{ + var options = sp.GetRequiredService>().Value; + return new PlanRegistry(options); +}); +builder.Services.AddSingleton(); + +builder.Services.AddHealthChecks().AddCheck("self", () => Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckResult.Healthy()); + +builder.Services.AddOpenTelemetry() + .WithMetrics(metricsBuilder => + { + metricsBuilder.AddMeter(RegistryTokenMetrics.MeterName); + metricsBuilder.AddAspNetCoreInstrumentation(); + metricsBuilder.AddRuntimeInstrumentation(); + }); + +builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: null, + configure: resourceOptions => + { + resourceOptions.Authority = bootstrapOptions.Authority.Issuer; + resourceOptions.RequireHttpsMetadata = bootstrapOptions.Authority.RequireHttpsMetadata; + resourceOptions.MetadataAddress = bootstrapOptions.Authority.MetadataAddress; + + resourceOptions.Audiences.Clear(); + foreach (var audience in bootstrapOptions.Authority.Audiences) + { + resourceOptions.Audiences.Add(audience); + } + }); + +builder.Services.AddAuthorization(options => +{ + var scopes = bootstrapOptions.Authority.RequiredScopes.Count == 0 + ? new[] { "registry.token.issue" } + : bootstrapOptions.Authority.RequiredScopes.ToArray(); + + options.AddPolicy("registry.token.issue", policy => + { + policy.RequireAuthenticatedUser(); + policy.Requirements.Add(new StellaOpsScopeRequirement(scopes)); + policy.AddAuthenticationSchemes(StellaOpsAuthenticationDefaults.AuthenticationScheme); + }); +}); + +var app = builder.Build(); + +app.UseSerilogRequestLogging(); +app.UseAuthentication(); +app.UseAuthorization(); + +app.MapHealthChecks("/healthz"); + +app.MapGet("/token", ( + HttpContext context, + [FromServices] IOptions options, + [FromServices] RegistryTokenIssuer issuer) => +{ + var serviceOptions = options.Value; + + var service = context.Request.Query["service"].FirstOrDefault()?.Trim(); + if (string.IsNullOrWhiteSpace(service)) + { + return Results.Problem( + detail: "The 'service' query parameter is required.", + statusCode: StatusCodes.Status400BadRequest); + } + + if (serviceOptions.Registry.AllowedServices.Count > 0 && + !serviceOptions.Registry.AllowedServices.Contains(service, StringComparer.OrdinalIgnoreCase)) + { + return Results.Problem( + detail: "The requested registry service is not permitted for this installation.", + statusCode: StatusCodes.Status403Forbidden); + } + + IReadOnlyList accessRequests; + try + { + accessRequests = RegistryScopeParser.Parse(context.Request.Query); + } + catch (InvalidScopeException ex) + { + return Results.Problem( + detail: ex.Message, + statusCode: StatusCodes.Status400BadRequest); + } + + if (accessRequests.Count == 0) + { + return Results.Problem( + detail: "At least one scope must be requested.", + statusCode: StatusCodes.Status400BadRequest); + } + + try + { + var response = issuer.IssueToken(context.User, service, accessRequests); + + return Results.Json(new + { + token = response.Token, + expires_in = response.ExpiresIn, + issued_at = response.IssuedAt.UtcDateTime.ToString("O"), + issued_token_type = "urn:ietf:params:oauth:token-type:access_token" + }); + } + catch (RegistryTokenException ex) + { + return Results.Problem( + detail: ex.Message, + statusCode: StatusCodes.Status403Forbidden); + } +}) +.WithName("GetRegistryToken") +.RequireAuthorization("registry.token.issue") +.Produces(StatusCodes.Status200OK) +.ProducesProblem(StatusCodes.Status400BadRequest) +.ProducesProblem(StatusCodes.Status403Forbidden); + +app.Run(); diff --git a/src/StellaOps.Registry.TokenService/Properties/launchSettings.json b/src/StellaOps.Registry.TokenService/Properties/launchSettings.json new file mode 100644 index 00000000..96048292 --- /dev/null +++ b/src/StellaOps.Registry.TokenService/Properties/launchSettings.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5068", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/StellaOps.Registry.TokenService/RegistryAccessModels.cs b/src/StellaOps.Registry.TokenService/RegistryAccessModels.cs new file mode 100644 index 00000000..63767922 --- /dev/null +++ b/src/StellaOps.Registry.TokenService/RegistryAccessModels.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace StellaOps.Registry.TokenService; + +/// +/// Represents a scope access request parsed from the scope query parameter. +/// +public sealed record RegistryAccessRequest(string Type, string Name, IReadOnlyList Actions); + +/// +/// Authorization decision. +/// +public sealed record RegistryAccessDecision(bool Allowed, string? FailureReason = null); diff --git a/src/StellaOps.Registry.TokenService/RegistryScopeParser.cs b/src/StellaOps.Registry.TokenService/RegistryScopeParser.cs new file mode 100644 index 00000000..2770989e --- /dev/null +++ b/src/StellaOps.Registry.TokenService/RegistryScopeParser.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Registry.TokenService; + +public static class RegistryScopeParser +{ + public static IReadOnlyList Parse(IQueryCollection query) + { + ArgumentNullException.ThrowIfNull(query); + + var scopes = new List(); + + if (query.TryGetValue("scope", out var scopeValues)) + { + foreach (var scope in scopeValues) + { + if (string.IsNullOrWhiteSpace(scope)) + { + continue; + } + + // Support space-delimited scopes per OAuth2 spec + foreach (var component in scope.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + scopes.Add(component); + } + } + } + + var requests = new List(scopes.Count); + foreach (var scope in scopes) + { + var request = ParseScope(scope); + requests.Add(request); + } + + return requests; + } + + private static RegistryAccessRequest ParseScope(string scope) + { + var segments = scope.Split(':', StringSplitOptions.TrimEntries); + if (segments.Length < 1) + { + throw new InvalidScopeException(scope, "scope missing resource type"); + } + + var type = segments[0]; + if (!string.Equals(type, "repository", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidScopeException(scope, $"unsupported resource type '{type}'"); + } + + if (segments.Length < 2 || string.IsNullOrWhiteSpace(segments[1])) + { + throw new InvalidScopeException(scope, "repository scope missing name"); + } + + var name = segments[1]; + var actions = segments.Length >= 3 && !string.IsNullOrWhiteSpace(segments[2]) + ? segments[2].Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + : Array.Empty(); + + if (actions.Length == 0) + { + actions = new[] { "pull" }; + } + + var normalized = actions + .Select(action => action.ToLowerInvariant()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return new RegistryAccessRequest(type.ToLowerInvariant(), name, normalized); + } +} + +public sealed class InvalidScopeException : Exception +{ + public InvalidScopeException(string scope, string reason) + : base($"Invalid scope '{scope}': {reason}") + { + Scope = scope; + Reason = reason; + } + + public string Scope { get; } + + public string Reason { get; } +} diff --git a/src/StellaOps.Registry.TokenService/RegistryTokenIssuer.cs b/src/StellaOps.Registry.TokenService/RegistryTokenIssuer.cs new file mode 100644 index 00000000..aa7eaa40 --- /dev/null +++ b/src/StellaOps.Registry.TokenService/RegistryTokenIssuer.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.IdentityModel.Tokens.Jwt; +using System.Linq; +using System.Security.Claims; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Registry.TokenService.Observability; +using StellaOps.Registry.TokenService.Security; + +namespace StellaOps.Registry.TokenService; + +public sealed class RegistryTokenIssuer +{ + private readonly RegistryTokenServiceOptions _options; + private readonly PlanRegistry _planRegistry; + private readonly RegistryTokenMetrics _metrics; + private readonly SigningCredentials _signingCredentials; + private readonly JwtSecurityTokenHandler _tokenHandler = new(); + private readonly TimeProvider _timeProvider; + + public RegistryTokenIssuer( + IOptions options, + PlanRegistry planRegistry, + RegistryTokenMetrics metrics, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(planRegistry); + ArgumentNullException.ThrowIfNull(metrics); + ArgumentNullException.ThrowIfNull(timeProvider); + + _options = options.Value; + _planRegistry = planRegistry; + _metrics = metrics; + _timeProvider = timeProvider; + _signingCredentials = SigningKeyLoader.Load(_options.Signing); + } + + public RegistryTokenResponse IssueToken( + ClaimsPrincipal principal, + string service, + IReadOnlyList requests) + { + var decision = _planRegistry.Authorize(principal, requests); + if (!decision.Allowed) + { + _metrics.TokensRejected.Add(1, new KeyValuePair("reason", decision.FailureReason ?? "denied")); + throw new RegistryTokenException(decision.FailureReason ?? "denied"); + } + + var now = _timeProvider.GetUtcNow(); + var expires = now + _options.Signing.Lifetime; + var subject = principal.FindFirstValue(ClaimTypes.NameIdentifier) + ?? principal.FindFirstValue("client_id") + ?? principal.FindFirstValue("sub") + ?? "anonymous"; + + var payload = new JwtPayload( + issuer: _options.Signing.Issuer, + audience: _options.Signing.Audience ?? service, + claims: null, + notBefore: now.UtcDateTime, + expires: expires.UtcDateTime, + issuedAt: now.UtcDateTime) + { + { JwtRegisteredClaimNames.Sub, subject }, + { JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString("n") }, + { "service", service }, + { "access", BuildAccessClaim(requests) } + }; + + var licenseId = principal.FindFirstValue("stellaops:license"); + if (!string.IsNullOrWhiteSpace(licenseId)) + { + payload["stellaops:license"] = licenseId; + } + + var token = new JwtSecurityToken(new JwtHeader(_signingCredentials), payload); + var serialized = _tokenHandler.WriteToken(token); + + var plan = principal.FindFirstValue("stellaops:plan") ?? _options.DefaultPlan ?? "unknown"; + _metrics.TokensIssued.Add(1, new KeyValuePair("plan", plan)); + + return new RegistryTokenResponse( + serialized, + (int)_options.Signing.Lifetime.TotalSeconds, + now); + } + + private static object BuildAccessClaim(IReadOnlyList requests) + { + return requests + .Select(request => new Dictionary + { + ["type"] = request.Type, + ["name"] = request.Name, + ["actions"] = request.Actions + }) + .ToArray(); + } +} + +public sealed class RegistryTokenResponse +{ + public RegistryTokenResponse(string token, int expiresInSeconds, DateTimeOffset issuedAt) + { + Token = token; + ExpiresIn = expiresInSeconds; + IssuedAt = issuedAt; + } + + public string Token { get; } + + public int ExpiresIn { get; } + + public DateTimeOffset IssuedAt { get; } +} + +public sealed class RegistryTokenException : Exception +{ + public RegistryTokenException(string reason) + : base($"Token request denied: {reason}") + { + Reason = reason; + } + + public string Reason { get; } +} diff --git a/src/StellaOps.Registry.TokenService/RegistryTokenServiceOptions.cs b/src/StellaOps.Registry.TokenService/RegistryTokenServiceOptions.cs new file mode 100644 index 00000000..ec9f90dc --- /dev/null +++ b/src/StellaOps.Registry.TokenService/RegistryTokenServiceOptions.cs @@ -0,0 +1,321 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace StellaOps.Registry.TokenService; + +/// +/// Strongly typed options for the registry token service. +/// +public sealed class RegistryTokenServiceOptions +{ + public const string SectionName = "RegistryTokenService"; + + /// + /// Authority validation options. + /// + public AuthorityOptions Authority { get; set; } = new(); + + /// + /// JWT signing options. + /// + public SigningOptions Signing { get; set; } = new(); + + /// + /// Registry-scoped settings. + /// + public RegistryOptions Registry { get; set; } = new(); + + /// + /// Plan catalogue. + /// + public IList Plans { get; set; } = new List(); + + /// + /// Identifiers that are revoked (license IDs or customer IDs). + /// + public IList RevokedLicenses { get; set; } = new List(); + + /// + /// Optional explicit default plan when no plan claim is supplied. + /// + public string? DefaultPlan { get; set; } + + public void Validate() + { + Authority.Validate(); + Signing.Validate(); + Registry.Validate(); + + if (Plans.Count == 0) + { + throw new InvalidOperationException("At least one plan rule must be configured."); + } + + foreach (var plan in Plans) + { + plan.Validate(); + } + + NormalizeList(RevokedLicenses, toLower: true); + + if (!string.IsNullOrWhiteSpace(DefaultPlan)) + { + var normalized = DefaultPlan.Trim(); + if (!Plans.Any(plan => string.Equals(plan.Name, normalized, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException($"Default plan '{normalized}' is not present in the plan catalogue."); + } + + DefaultPlan = normalized; + } + } + + private static void NormalizeList(IList values, bool toLower) + { + if (values.Count == 0) + { + return; + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var index = values.Count - 1; index >= 0; index--) + { + var value = values[index]; + if (string.IsNullOrWhiteSpace(value)) + { + values.RemoveAt(index); + continue; + } + + var normalized = value.Trim(); + if (toLower) + { + normalized = normalized.ToLowerInvariant(); + } + + if (!seen.Add(normalized)) + { + values.RemoveAt(index); + continue; + } + + values[index] = normalized; + } + } + + public sealed class AuthorityOptions + { + /// + /// Issuer/authority URL (e.g. https://authority.stella.internal). + /// + public string Issuer { get; set; } = string.Empty; + + /// + /// Optional explicit metadata (JWKS) endpoint. + /// + public string? MetadataAddress { get; set; } + + /// + /// Whether HTTPS metadata is required (disabled for dev loops). + /// + public bool RequireHttpsMetadata { get; set; } = true; + + /// + /// Audiences that resource server accepts. + /// + public IList Audiences { get; set; } = new List(); + + /// + /// Scopes required to hit the token endpoint. + /// + public IList RequiredScopes { get; set; } = new List { "registry.token.issue" }; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Issuer)) + { + throw new InvalidOperationException("Authority issuer must be configured."); + } + + if (!Uri.TryCreate(Issuer.Trim(), UriKind.Absolute, out var uri)) + { + throw new InvalidOperationException("Authority issuer must be an absolute URI."); + } + + if (RequireHttpsMetadata && + !uri.IsLoopback && + !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Authority issuer must use HTTPS when RequireHttpsMetadata is true."); + } + + NormalizeList(Audiences, toLower: false); + NormalizeList(RequiredScopes, toLower: true); + } + } + + public sealed class SigningOptions + { + /// + /// Issuer for generated registry tokens. + /// + public string Issuer { get; set; } = string.Empty; + + /// + /// Optional audience override. Defaults to the requested registry service. + /// + public string? Audience { get; set; } + + /// + /// Path to an RSA private key (PEM or PFX). + /// + public string KeyPath { get; set; } = string.Empty; + + /// + /// Optional password when loading a PFX. + /// + public string? KeyPassword { get; set; } + + /// + /// Optional key identifier (kid) appended to the JWT header. + /// + public string? KeyId { get; set; } + + /// + /// Token lifetime. + /// + public TimeSpan Lifetime { get; set; } = TimeSpan.FromMinutes(5); + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Issuer)) + { + throw new InvalidOperationException("Signing.Issuer must be provided."); + } + + if (Lifetime <= TimeSpan.Zero || Lifetime > TimeSpan.FromHours(1)) + { + throw new InvalidOperationException("Signing.Lifetime must be between 1 second and 1 hour."); + } + + if (string.IsNullOrWhiteSpace(KeyPath)) + { + throw new InvalidOperationException("Signing.KeyPath must be configured."); + } + + var file = KeyPath.Trim(); + if (!Path.IsPathRooted(file)) + { + file = Path.GetFullPath(file); + } + + if (!File.Exists(file)) + { + throw new InvalidOperationException($"Signing.KeyPath '{file}' does not exist."); + } + + KeyPath = file; + if (!string.IsNullOrWhiteSpace(KeyId)) + { + KeyId = KeyId.Trim(); + } + } + } + + public sealed class RegistryOptions + { + /// + /// Registry service realm (matches Docker registry configuration). + /// + public string Realm { get; set; } = string.Empty; + + /// + /// Allowed service identifiers. Empty list permits any service. + /// + public IList AllowedServices { get; set; } = new List(); + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Realm)) + { + throw new InvalidOperationException("Registry.Realm must be provided."); + } + + if (!Uri.TryCreate(Realm.Trim(), UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Registry.Realm must be an absolute URI."); + } + + NormalizeList(AllowedServices, toLower: false); + } + } + + public sealed class PlanRule + { + /// + /// Plan identifier (case-insensitive). + /// + public string Name { get; set; } = string.Empty; + + /// + /// Repository rules associated to the plan. + /// + public IList Repositories { get; set; } = new List(); + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Name)) + { + throw new InvalidOperationException("Plan name cannot be empty."); + } + + Name = Name.Trim(); + + if (Repositories.Count == 0) + { + throw new InvalidOperationException($"Plan '{Name}' must specify at least one repository rule."); + } + + foreach (var repo in Repositories) + { + repo.Validate(Name); + } + } + } + + public sealed class RepositoryRule + { + /// + /// Repository pattern (supports '*' wildcard). + /// + public string Pattern { get; set; } = string.Empty; + + /// + /// Allowed actions (pull/push/delete, etc.) + /// + public IList Actions { get; set; } = new List { "pull" }; + + public void Validate(string planName) + { + if (string.IsNullOrWhiteSpace(Pattern)) + { + throw new InvalidOperationException($"Plan '{planName}' contains a repository rule with an empty pattern."); + } + + Pattern = Pattern.Trim(); + if (Pattern.Contains(' ', StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Plan '{planName}' repository pattern '{Pattern}' may not contain spaces."); + } + + if (Actions.Count == 0) + { + throw new InvalidOperationException($"Plan '{planName}' repository '{Pattern}' must define allowed actions."); + } + + NormalizeList(Actions, toLower: true); + } + } +} diff --git a/src/StellaOps.Registry.TokenService/Security/SigningKeyLoader.cs b/src/StellaOps.Registry.TokenService/Security/SigningKeyLoader.cs new file mode 100644 index 00000000..ee464d5c --- /dev/null +++ b/src/StellaOps.Registry.TokenService/Security/SigningKeyLoader.cs @@ -0,0 +1,66 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Registry.TokenService.Security; + +internal static class SigningKeyLoader +{ + public static SigningCredentials Load(RegistryTokenServiceOptions.SigningOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + SecurityKey key; + + var extension = Path.GetExtension(options.KeyPath); + if (string.Equals(extension, ".pfx", StringComparison.OrdinalIgnoreCase)) + { + key = LoadFromPfx(options.KeyPath, options.KeyPassword); + } + else + { + key = LoadFromPem(options.KeyPath); + } + + var credentials = new SigningCredentials(key, SecurityAlgorithms.RsaSha256) + { + CryptoProviderFactory = new CryptoProviderFactory { CacheSignatureProviders = true } + }; + + if (!string.IsNullOrWhiteSpace(options.KeyId)) + { + credentials.Key.KeyId = options.KeyId; + } + + return credentials; + } + + private static SecurityKey LoadFromPfx(string path, string? password) + { + using var cert = X509CertificateLoader.LoadPkcs12FromFile(path, password, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.EphemeralKeySet); + if (!cert.HasPrivateKey) + { + throw new InvalidOperationException($"Certificate '{path}' does not contain a private key."); + } + + if (cert.GetRSAPrivateKey() is not RSA rsa) + { + throw new InvalidOperationException($"Certificate '{path}' does not contain an RSA private key."); + } + + var parameters = rsa.ExportParameters(true); + rsa.Dispose(); + + return new RsaSecurityKey(parameters) { KeyId = cert.Thumbprint }; + } + + private static SecurityKey LoadFromPem(string path) + { + using var rsa = RSA.Create(); + var pem = File.ReadAllText(path); + rsa.ImportFromPem(pem); + return new RsaSecurityKey(rsa.ExportParameters(includePrivateParameters: true)); + } +} diff --git a/src/StellaOps.Registry.TokenService/StellaOps.Registry.TokenService.csproj b/src/StellaOps.Registry.TokenService/StellaOps.Registry.TokenService.csproj new file mode 100644 index 00000000..f86c30fe --- /dev/null +++ b/src/StellaOps.Registry.TokenService/StellaOps.Registry.TokenService.csproj @@ -0,0 +1,24 @@ + + + net10.0 + enable + enable + preview + true + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Registry.TokenService/appsettings.Development.json b/src/StellaOps.Registry.TokenService/appsettings.Development.json new file mode 100644 index 00000000..0c208ae9 --- /dev/null +++ b/src/StellaOps.Registry.TokenService/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/src/StellaOps.Registry.TokenService/appsettings.json b/src/StellaOps.Registry.TokenService/appsettings.json new file mode 100644 index 00000000..10f68b8c --- /dev/null +++ b/src/StellaOps.Registry.TokenService/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/src/StellaOps.RiskEngine/StellaOps.RiskEngine.WebService/StellaOps.RiskEngine.WebService.csproj b/src/StellaOps.RiskEngine/StellaOps.RiskEngine.WebService/StellaOps.RiskEngine.WebService.csproj index fb13a6b6..b870c717 100644 --- a/src/StellaOps.RiskEngine/StellaOps.RiskEngine.WebService/StellaOps.RiskEngine.WebService.csproj +++ b/src/StellaOps.RiskEngine/StellaOps.RiskEngine.WebService/StellaOps.RiskEngine.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.RiskEngine/StellaOps.RiskEngine.Worker/StellaOps.RiskEngine.Worker.csproj b/src/StellaOps.RiskEngine/StellaOps.RiskEngine.Worker/StellaOps.RiskEngine.Worker.csproj index 7da8df6e..667832a0 100644 --- a/src/StellaOps.RiskEngine/StellaOps.RiskEngine.Worker/StellaOps.RiskEngine.Worker.csproj +++ b/src/StellaOps.RiskEngine/StellaOps.RiskEngine.Worker/StellaOps.RiskEngine.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.SbomService/TASKS.md b/src/StellaOps.SbomService/TASKS.md index e4b8850b..c19e6c6c 100644 --- a/src/StellaOps.SbomService/TASKS.md +++ b/src/StellaOps.SbomService/TASKS.md @@ -2,7 +2,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SBOM-SERVICE-21-001 | TODO | SBOM Service Guild, Cartographer Guild | CONCELIER-GRAPH-21-001 | Publish normalized SBOM projection schema (components, relationships, scopes, entrypoints) and implement read API with pagination + tenant enforcement. | Schema validated with fixtures; API documented; integration tests cover CycloneDX/SPDX inputs. | -| SBOM-SERVICE-21-002 | TODO | SBOM Service Guild, Scheduler Guild | SBOM-SERVICE-21-001, SCHED-MODELS-21-001 | Emit change events (`sbom.version.created`) carrying digest/version metadata for Cartographer builds; add replay/backfill tooling. | Events published on new SBOMs; consumer harness validated; replay scripts documented. | +| SBOM-SERVICE-21-002 | TODO | SBOM Service Guild, Scheduler Guild | SBOM-SERVICE-21-001, SCHED-MODELS-21-001 | Emit change events (`sbom.version.created`) carrying digest/version metadata for Graph Indexer builds; add replay/backfill tooling. | Events published on new SBOMs; consumer harness validated; replay scripts documented. | | SBOM-SERVICE-21-003 | TODO | SBOM Service Guild | SBOM-SERVICE-21-001 | Provide entrypoint/service node management API (list/update overrides) feeding Cartographer path relevance with deterministic defaults. | Entrypoint API live; overrides persisted; docs updated; tests cover fallback logic. | | SBOM-SERVICE-21-004 | TODO | SBOM Service Guild, Observability Guild | SBOM-SERVICE-21-001 | Wire observability: metrics (`sbom_projection_seconds`, `sbom_projection_size`), traces, structured logs with tenant info; set alerts for backlog. | Metrics/traces exposed; dashboards updated; alert thresholds defined. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Deno/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Deno/TASKS.md new file mode 100644 index 00000000..448e55d9 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Deno/TASKS.md @@ -0,0 +1,21 @@ +# Deno Analyzer Task Board +> **Imposed rule:** work of this type or tasks of this type on this component — and everywhere else it should be applied. + +## Deno Entry-Point Analyzer (Sprint 49) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-DENO-26-001 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309N | Build input normalizer & VFS for Deno projects: merge `deno.json(c)`, import maps, lockfiles, vendor dirs, `$DENO_DIR` caches, and container layers. Detect runtime/toolchain hints deterministically. | Normalizer ingests fixtures (source+vendor, cache-only, container) without network; outputs config digest, import map, cache locations, and deterministic module root inventory. | +| SCANNER-ANALYZERS-DENO-26-002 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-001 | Module graph builder: resolve static/dynamic imports using import map, `deno.lock`, vendor/, cache, npm bridge, node: builtins, WASM/JSON assertions. Annotate edges with resolution source and form. | Graph reconstruction succeeds on fixtures (vendor, cache-only, npm, remote). Edges include `form`, `source`, `resolution` (`vendor|cache|fs|declared_only`). Determinism harness passes. | +| SCANNER-ANALYZERS-DENO-26-003 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-002 | NPM/Node compat adapter: map `npm:` specifiers to cached packages or compat `node_modules`, evaluate package `exports`/conditions, record node: builtin usage. | Fixtures with npm bridge resolve to cached/vendor modules; outputs include npm package metadata + node builtin list; unresolved npm deps flagged. | +| SCANNER-ANALYZERS-DENO-26-004 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-002 | Static analyzer for permission/capability signals (FS, net, env, process, crypto, FFI, workers). Detect dynamic-import patterns, literal fetch URLs, tasks vs declared permissions. | Capability records emitted with evidence snippets; dynamic import warnings include pattern info; task vs inferred permission diffs reported. | +| SCANNER-ANALYZERS-DENO-26-005 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-002 | Bundle/binary inspector: parse eszip bundles and `deno compile` executables (embedded eszip + snapshot) to recover module graph, config, embedded resources. | Bundle and compile fixtures yield recovered module lists, digests, and target metadata; compiled exe scanning <600ms; determinism verified. | +| SCANNER-ANALYZERS-DENO-26-006 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-002 | Container adapter: traverse OCI layers for `deno`, caches, vendor directories, compiled binaries; merge module provenance with layer info. | Container fixtures output runtime version, cache roots, vendor mapping, binary metadata with layer provenance; determinism maintained. | + +## Deno Observation & Runtime (Sprint 50) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-DENO-26-007 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-002 | Produce AOC-compliant observations: entrypoints, modules, edges, permissions, workers, warnings, binaries with reason codes and contexts. | Observation JSON for fixtures deterministic; edges include form/source/reason; capabilities and permission drift recorded; passes AOC lint. | +| SCANNER-ANALYZERS-DENO-26-008 | TODO | Deno Analyzer Guild, QA Guild | SCANNER-ANALYZERS-DENO-26-007 | Fixture suite + performance benchmarks (vendor, npm, FFI, workers, dynamic import, bundle/binary, cache-only, container). | Fixture set under `fixtures/lang/deno/ep`; determinism and perf (<1.5s 2k-module graph) CI gates enabled. | +| SCANNER-ANALYZERS-DENO-26-009 | TODO | Deno Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-DENO-26-007 | Optional runtime evidence hooks (loader/require shim) capturing module loads + permissions during harnessed execution with path hashing. | Runtime harness logs module loads for sample app with scrubbed paths; runtime edges merge without altering static precedence; privacy doc updated. | +| SCANNER-ANALYZERS-DENO-26-010 | TODO | Deno Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-DENO-26-007 | Package analyzer plug-in, add CLI (`stella deno inspect|resolve|trace`) commands, update Offline Kit docs, ensure Worker integration. | Plug-in manifest deployed; CLI commands documented/tested; Offline Kit instructions updated; worker restart verified. | +| SCANNER-ANALYZERS-DENO-26-011 | TODO | Deno Analyzer Guild | SCANNER-ANALYZERS-DENO-26-004 | Policy signal emitter: net/fs/env/ffi/process/crypto capabilities, remote origin list, npm usage, wasm modules, dynamic-import warnings. | Outputs include policy signal section consumed by tests; schema documented; sample policy evaluation validated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md index 00d81dc5..5f1f763d 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md @@ -1,10 +1,31 @@ -# Node Analyzer Task Flow - -| Seq | ID | Status | Depends on | Description | Exit Criteria | -|-----|----|--------|------------|-------------|---------------| -| 1 | SCANNER-ANALYZERS-LANG-10-302A | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-307 | Build deterministic module graph walker covering npm, Yarn, and PNPM; capture package.json provenance and integrity metadata. | Walker indexes >100 k modules in <1.5 s (hot cache); golden fixtures verify deterministic ordering and path normalization. | -| 2 | SCANNER-ANALYZERS-LANG-10-302B | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-302A | Resolve workspaces/symlinks and attribute components to originating package with usage hints; guard against directory traversal. | Workspace attribution accurate on multi-workspace fixture; symlink resolver proves canonical path; security tests ensure no traversal. | -| 3 | SCANNER-ANALYZERS-LANG-10-302C | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-302B | Surface script metadata (postinstall/preinstall) and policy hints; emit telemetry counters and evidence records. | Analyzer output includes script metadata + evidence; metrics `scanner_analyzer_node_scripts_total` recorded; policy hints documented. | +# Node Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| +| 1 | SCANNER-ANALYZERS-LANG-10-302A | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-307 | Build deterministic module graph walker covering npm, Yarn, and PNPM; capture package.json provenance and integrity metadata. | Walker indexes >100 k modules in <1.5 s (hot cache); golden fixtures verify deterministic ordering and path normalization. | +| 2 | SCANNER-ANALYZERS-LANG-10-302B | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-302A | Resolve workspaces/symlinks and attribute components to originating package with usage hints; guard against directory traversal. | Workspace attribution accurate on multi-workspace fixture; symlink resolver proves canonical path; security tests ensure no traversal. | +| 3 | SCANNER-ANALYZERS-LANG-10-302C | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-302B | Surface script metadata (postinstall/preinstall) and policy hints; emit telemetry counters and evidence records. | Analyzer output includes script metadata + evidence; metrics `scanner_analyzer_node_scripts_total` recorded; policy hints documented. | | 4 | SCANNER-ANALYZERS-LANG-10-307N | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-302C | Integrate shared helpers for license/licence evidence, canonical JSON serialization, and usage flag propagation. | Reuse shared helpers without duplication; unit tests confirm stable metadata merge; no analyzer-specific serializer drift. | | 5 | SCANNER-ANALYZERS-LANG-10-308N | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-307N | Author determinism harness + fixtures for Node analyzer; add benchmark suite. | Fixtures committed under `Fixtures/lang/node/`; determinism CI job compares JSON snapshots; benchmark CSV published. | | 6 | SCANNER-ANALYZERS-LANG-10-309N | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-308N | Package Node analyzer as restart-time plug-in (manifest, DI registration, Offline Kit notes). | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer after restart; Offline Kit docs updated. | + +## Node Entry-Point Analyzer (Sprint 41) +> **Imposed rule:** work of this type or tasks of this type on this component — and everywhere else it should be applied. +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-NODE-22-001 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309N | Build input normalizer + VFS for Node projects: dirs, tgz, container layers, pnpm store, Yarn PnP zips; detect Node version targets (`.nvmrc`, `.node-version`, Dockerfile) and workspace roots deterministically. | Normalizer handles fixtures (npm, pnpm, Yarn classic/PnP, container) without extraction; records node_version, workspace list, and symlink mode with golden outputs. | +| SCANNER-ANALYZERS-NODE-22-002 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-001 | Implement entrypoint discovery (bin/main/module/exports/imports, workers, electron, shebang scripts) and condition set builder per entrypoint. | Entrypoint inventory generated for fixtures (library, CLI, electron, worker); each entrypoint includes kind, start file, conditions; determinism harness updated. | +| SCANNER-ANALYZERS-NODE-22-003 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-001 | Parse JS/TS sources for static `import`, `require`, `import()` and string concat cases; flag dynamic patterns with confidence levels; support source map de-bundling. | Static edges + dynamic-specifier warnings emitted for fixtures (ESM, CJS, bundles); source map fixture rewrites concatenated modules with `confidence` metadata. | +| SCANNER-ANALYZERS-NODE-22-004 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-002 | Implement Node resolver engine for CJS + ESM (core modules, exports/imports maps, conditions, extension priorities, self-references) parameterised by node_version. | Resolver reproduces Node 18/20 semantics across fixture matrix; includes explain trace per edge; unit tests cover exports/conditions/extension ordering. | +| SCANNER-ANALYZERS-NODE-22-005 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-004 | Add package manager adapters: Yarn PnP (.pnp.data/.pnp.cjs), pnpm virtual store, npm/Yarn classic hoists; operate entirely in virtual FS. | PnP fixture resolves via `.pnp.data.json`/cache zips; pnpm fixture follows `.pnpm` symlinks; classic hoist fixture maintains first-wins ordering; warnings emitted for unreadable PnP. | +| SCANNER-ANALYZERS-NODE-22-006 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-004 | Detect bundles + source maps, reconstruct module specifiers, and correlate to original paths; support dual CJS/ESM graphs with conditions. | Bundle fixture using source maps produces `bundle-map` edges with medium confidence and original source paths; dual package fixture yields separate import/require graphs. | +| SCANNER-ANALYZERS-NODE-22-007 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-002 | Scan for native addons (.node), WASM modules, and core capability signals (child_process, vm, worker_threads); emit hint edges and native metadata. | Fixtures with native addon/WASM produce `native-addon`/`wasm` edges plus capability hints; metadata captures ABI, N-API, OS/arch where available. | + +## Node Observation & Runtime (Sprint 42) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-NODE-22-008 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-004 | Produce AOC-compliant observations: entrypoints, components (pkg/native/wasm), edges (esm-import, cjs-require, exports, json, native-addon, wasm, worker) with reason codes/confidence and resolver traces. | Observation JSON for fixtures deterministic; edges include reason codes + confidence; resolves attach trace data; passes AOC lint. | +| SCANNER-ANALYZERS-NODE-22-009 | TODO | Node Analyzer Guild, QA Guild | SCANNER-ANALYZERS-NODE-22-008 | Author fixture suite + performance benchmarks (npm, pnpm, PnP, bundle, electron, worker) with golden outputs and latency budgets. | Fixtures stored under `fixtures/lang/node/ep`; determinism + perf (<350ms npm, <900ms PnP) enforced via CI benchmarks. | +| SCANNER-ANALYZERS-NODE-22-010 | TODO | Node Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-NODE-22-008 | Implement optional runtime evidence hooks (ESM loader, CJS require hook) with path scrubbing and loader ID hashing; emit runtime-* edges. | Sandbox harness records runtime resolve events for sample app; paths hashed; runtime edges merge with static graph without altering first-wins selection. | +| SCANNER-ANALYZERS-NODE-22-011 | TODO | Node Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-NODE-22-008 | Package updated analyzer as restart-time plug-in, expose Scanner CLI (`stella node *`) commands, refresh Offline Kit documentation. | Plugins folder updated; worker loads analyzer; CLI commands documented and smoke-tested; Offline Kit instructions include Node analyzer usage. | +| SCANNER-ANALYZERS-NODE-22-012 | TODO | Node Analyzer Guild | SCANNER-ANALYZERS-NODE-22-001 | Integrate container filesystem adapter (OCI layers, Dockerfile hints) and record NODE_OPTIONS/env warnings. | Container fixture parsed: Node base image + NODE_OPTIONS captured; entrypoints resolved relative to image root; warnings emitted for loader flags. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Php/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Php/TASKS.md new file mode 100644 index 00000000..45970ef2 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Php/TASKS.md @@ -0,0 +1,22 @@ +# PHP Analyzer Task Board +> **Imposed rule:** work of this type or tasks of this type on this component — and everywhere else it should be applied. + +## PHP Entry-Point Analyzer (Sprint 51) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-PHP-27-001 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309P | Build input normalizer & VFS for PHP projects: merge source trees, composer manifests, vendor/, php.ini/conf.d, `.htaccess`, FPM configs, container layers. Detect framework/CMS fingerprints deterministically. | Normalizer ingests fixtures (Laravel, Symfony, WordPress, Drupal, container) without network; outputs config inventory, framework tags, and deterministic module/vendor root list. | +| SCANNER-ANALYZERS-PHP-27-002 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-001 | Composer/Autoload analyzer: parse composer.json/lock/installed.json, generate package nodes, autoload edges (psr-4/0/classmap/files), bin entrypoints, composer plugins. | Composer fixtures produce package list with PURLs, autoload graph, bin scripts, plugin inventory; determinism harness passes. | +| SCANNER-ANALYZERS-PHP-27-003 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-002 | Include/require graph builder: resolve static includes, capture dynamic include patterns, bootstrap chains, merge with autoload edges. | Include graph constructed for fixtures (legacy, WordPress, Laravel); dynamic includes recorded with patterns; deterministic ordering ensured. | +| SCANNER-ANALYZERS-PHP-27-004 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-003 | Runtime capability scanner: detect exec/fs/net/env/serialization/crypto/database usage, stream wrappers, uploads; record evidence snippets. | Capability signals generated for fixtures (exec, curl, unserialize); outputs include file/line/evidence hash; determinism validated. | +| SCANNER-ANALYZERS-PHP-27-005 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-001 | PHAR/Archive inspector: parse phar manifests/stubs, hash files, detect embedded vendor trees and phar:// usage. | PHAR fixtures yield file inventory, signature metadata, autoload edges; deterministic parse under <800 ms. | +| SCANNER-ANALYZERS-PHP-27-006 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-001 | Framework/CMS surface mapper: extract routes, controllers, middleware, CLI/cron entrypoints for Laravel/Symfony/Slim/WordPress/Drupal/Magento. | Framework fixtures produce route/endpoint lists, CLI command inventory, cron hints; tests validate coverage. | +| SCANNER-ANALYZERS-PHP-27-007 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-001 | Container & extension detector: parse php.ini/conf.d, map extensions to .so/.dll, collect web server/FPM settings, upload limits, disable_functions. | Container fixture outputs extension list with file paths, php.ini directives, web server front controller data; determinism maintained. | + +## PHP Observation & Runtime (Sprint 52) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-PHP-27-008 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-002 | Produce AOC-compliant observations: entrypoints, packages, extensions, modules, edges (require/autoload), capabilities, routes, configs. | Observation JSON for fixtures deterministic; edges contain reason/form; capability and route inventories included; passes AOC lint. | +| SCANNER-ANALYZERS-PHP-27-009 | TODO | PHP Analyzer Guild, QA Guild | SCANNER-ANALYZERS-PHP-27-008 | Fixture suite + performance benchmarks (Laravel, Symfony, WordPress, legacy, PHAR, container) with golden outputs. | Fixture set under `fixtures/lang/php/ep`; determinism and perf (<4s 50k files) gates active. | +| SCANNER-ANALYZERS-PHP-27-010 | TODO | PHP Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-PHP-27-008 | Optional runtime evidence hooks (if provided) to ingest audit logs or opcode cache stats with path hashing. | Runtime harness (if supplied) integrates without altering static precedence; hashed paths; documentation updated. | +| SCANNER-ANALYZERS-PHP-27-011 | TODO | PHP Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-PHP-27-008 | Package analyzer plug-in, add CLI (`stella php inspect|resolve`) commands, update Offline Kit docs, ensure Worker integration. | Plug-in manifest deployed; CLI commands documented/tested; Offline Kit instructions updated; worker restart verified. | +| SCANNER-ANALYZERS-PHP-27-012 | TODO | PHP Analyzer Guild | SCANNER-ANALYZERS-PHP-27-004 | Policy signal emitter: extension requirements/presence, dangerous constructs counters, stream wrapper usage, capability summaries. | Policy signal section emitted and validated against fixtures; schema documented; sample policy evaluation added. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md index 4d588ba7..f14e0cb1 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md @@ -1,10 +1,31 @@ -# Python Analyzer Task Flow - -| Seq | ID | Status | Depends on | Description | Exit Criteria | -|-----|----|--------|------------|-------------|---------------| +# Python Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| | 1 | SCANNER-ANALYZERS-LANG-10-303A | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-307 | STREAM-based parser for `*.dist-info` (`METADATA`, `WHEEL`, `entry_points.txt`) with normalization + evidence capture. | Parser handles CPython 3.8–3.12 metadata variations; fixtures confirm canonical ordering and UTF-8 handling. | | 2 | SCANNER-ANALYZERS-LANG-10-303B | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-303A | RECORD hash verifier with chunked hashing, Zip64 support, and mismatch diagnostics. | Verifier processes 5 GB RECORD fixture without allocations >2 MB; mismatches produce deterministic evidence records. | | 3 | SCANNER-ANALYZERS-LANG-10-303C | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-303B | Editable install + pip cache detection; integrate EntryTrace hints for runtime usage flags. | Editable installs resolved to source path; usage flags propagated; regression tests cover mixed editable + wheel installs. | | 4 | SCANNER-ANALYZERS-LANG-10-307P | DONE (2025-10-23) | SCANNER-ANALYZERS-LANG-10-303C | Shared helper integration (license metadata, quiet provenance, component merging). | Shared helpers reused; analyzer-specific metadata minimal; deterministic merge tests pass. | | 5 | SCANNER-ANALYZERS-LANG-10-308P | DONE (2025-10-23) | SCANNER-ANALYZERS-LANG-10-307P | Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. | Fixtures under `Fixtures/lang/python/`; determinism CI guard; benchmark CSV added with threshold alerts. | | 6 | SCANNER-ANALYZERS-LANG-10-309P | DONE (2025-10-23) | SCANNER-ANALYZERS-LANG-10-308P | Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer; Offline Kit doc updated. | + +## Python Entry-Point Analyzer (Sprint 43) +> **Imposed rule:** work of this type or tasks of this type on this component — and everywhere else it should be applied. +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-PYTHON-23-001 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309P | Build input normalizer & virtual filesystem for wheels, sdists, editable installs, zipapps, site-packages trees, and container roots. Detect Python version targets (`pyproject.toml`, `runtime.txt`, Dockerfile) + virtualenv layout deterministically. | Normalizer ingests fixtures (venv, wheel, sdist, zipapp, container layer) without extraction; records python_version, root metadata, and namespace resolution hints; determinism harness updated. | +| SCANNER-ANALYZERS-PYTHON-23-002 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-001 | Entrypoint discovery: module `__main__`, console_scripts entry points, `scripts`, zipapp main, `manage.py`/gunicorn/celery patterns. Capture invocation context (module vs package, argv wrappers). | Fixtures produce entrypoint list with kind (console, module, package, zipapp, framework) and deterministic ordering; warnings for missing targets recorded. | +| SCANNER-ANALYZERS-PYTHON-23-003 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-001 | Static import graph builder using AST and bytecode fallback. Support `import`, `from ... import`, relative imports, `importlib.import_module`, `__import__` with literal args, `pkgutil.extend_path`. | AST scanner emits edges for explicit imports; literal importlib calls covered; unresolved/dynamic patterns yield `dynamic-import` warnings with candidate prefixes; regression fixtures pass. | +| SCANNER-ANALYZERS-PYTHON-23-004 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-003 | Python resolver engine (importlib semantics) handling namespace packages (PEP 420), package discovery order, `.pth` files, `sys.path` composition, zipimport, and site-packages precedence across virtualenv/container roots. | Resolver reproduces importlib behaviour on fixture matrix (namespace pkg, zipimport, multi-site-dir); includes explain traces; determinism tests for path ordering succeed. | +| SCANNER-ANALYZERS-PYTHON-23-005 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-004 | Packaging adapters: pip editable (`.egg-link`), Poetry/Flit layout, Conda prefix, `.dist-info/RECORD` cross-check, container layer overlays. | Adapters resolve editable links, conda pkgs, layered site-packages; edges capture provider path + metadata; warnings emitted for missing RECORD entries. | +| SCANNER-ANALYZERS-PYTHON-23-006 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-003 | Detect native extensions (`*.so`, `*.pyd`), CFFI modules, ctypes loaders, embedded WASM, and runtime capability signals (subprocess, multiprocessing, ctypes, eval). | Fixtures with native/CFFI/ctypes emit `native-extension`, `cffi`, `ctypes` hints; capability flags recorded; metadata captures ABI/platform info. | +| SCANNER-ANALYZERS-PYTHON-23-007 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-002 | Framework/config heuristics: Django, Flask, FastAPI, Celery, AWS Lambda handlers, Gunicorn, Click/Typer CLIs, logging configs, pyproject optional dependencies. Tagged as hints only. | Framework fixtures produce hint records with source files (settings.py, pyproject extras, celery app); no resolver impact; determinism maintained. | + +## Python Observation & Runtime (Sprint 44) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-PYTHON-23-008 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-004 | Produce AOC-compliant observations: entrypoints, components (modules/packages/native), edges (import, namespace, dynamic-hint, native-extension) with reason codes/confidence and resolver traces. | Observation JSON for fixtures deterministic; includes explain trace per edge and namespace resolution metadata; passes AOC compliance lint. | +| SCANNER-ANALYZERS-PYTHON-23-009 | TODO | Python Analyzer Guild, QA Guild | SCANNER-ANALYZERS-PYTHON-23-008 | Fixture suite + perf benchmarks covering virtualenv, namespace packages, zipapp, editable installs, containers, lambda handler. | Fixture set committed under `fixtures/lang/python/ep`; determinism CI and perf (<250ms medium project) gates enabled. | +| SCANNER-ANALYZERS-PYTHON-23-010 | TODO | Python Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-PYTHON-23-008 | Optional runtime evidence: import hook capturing module load events with path scrubbing, optional bytecode instrumentation for `importlib` hooks, multiprocessing tracer. | Runtime harness records module loads for sample app; paths hashed; runtime edges merge without altering resolver precedence; privacy doc updated. | +| SCANNER-ANALYZERS-PYTHON-23-011 | TODO | Python Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-PYTHON-23-008 | Package analyzer plug-in, add CLI commands (`stella python inspect|resolve|trace`), update Offline Kit guidance. | Plugin manifest deployed; CLI commands documented & smoke tested; Offline Kit instructions cover Python analyzer usage; worker restart verified. | +| SCANNER-ANALYZERS-PYTHON-23-012 | TODO | Python Analyzer Guild | SCANNER-ANALYZERS-PYTHON-23-001 | Container/zipapp adapter enhancements: parse OCI layers for Python runtime, detect `PYTHONPATH`/`PYTHONHOME` env, record warnings for sitecustomize/startup hooks. | Container fixtures output runtime metadata (python binary, env vars) and warnings for startup hooks; zipapp fixture resolves internal modules; determinism retained. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md new file mode 100644 index 00000000..f2574e13 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md @@ -0,0 +1,22 @@ +# Ruby Analyzer Task Board +> **Imposed rule:** work of this type or tasks of this type on this component — and everywhere else it should be applied. + +## Ruby Entry-Point Analyzer (Sprint 53) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-RUBY-28-001 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309R | Build input normalizer & VFS for Ruby projects: merge source trees, Gemfile/Gemfile.lock, vendor/bundle, .gem archives, `.bundle/config`, Rack configs, containers. Detect framework/job fingerprints deterministically. | Normalizer ingests fixtures (Rails, Rack, Sinatra, Sidekiq, container) without network; outputs config inventory, framework tags, ruby version hints, deterministic gem/vendor root list. | +| SCANNER-ANALYZERS-RUBY-28-002 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-001 | Gem & Bundler analyzer: parse Gemfile/Gemfile.lock, vendor specs, .gem archives, produce package nodes (PURLs), dependency edges, bin scripts, Bundler group metadata. | Fixtures produce package list with version, groups, path/git sources; .gem archives decoded safely; determinism harness passes. | +| SCANNER-ANALYZERS-RUBY-28-003 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-002 | Require/autoload graph builder: resolve static/dynamic require, require_relative, load; infer Zeitwerk autoload paths and Rack boot chain. | Require graph built for fixtures (Rails, Rack, legacy); dynamic require warnings recorded; zeitwerk edges generated; deterministic ordering ensured. | +| SCANNER-ANALYZERS-RUBY-28-004 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-001 | Framework surface mapper: extract routes/controllers/middleware for Rails/Rack/Sinatra/Grape/Hanami; inventory jobs/schedulers (Sidekiq, Resque, ActiveJob, whenever, clockwork). | Framework fixtures emit route, controller, middleware, job, scheduler entries with provenance; tests validate coverage. | +| SCANNER-ANALYZERS-RUBY-28-005 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-003 | Capability analyzer: detect os-exec, filesystem, network, serialization, crypto, DB usage, TLS posture, dynamic eval; record evidence snippets with file/line. | Capability signals generated for fixtures (system, Net::HTTP, YAML.load, exec); outputs deterministic with hashed snippets. | +| SCANNER-ANALYZERS-RUBY-28-006 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-001 | Rake task & scheduler analyzer: parse Rakefiles/lib/tasks, capture task names/prereqs/shell commands; parse Sidekiq/whenever/clockwork configs into schedules. | Task/scheduler inventory produced for fixtures; includes cron specs, shell commands; determinism confirmed. | +| SCANNER-ANALYZERS-RUBY-28-007 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-001 | Container/runtime scanner: detect Ruby version, installed gems, native extensions, web server configs in OCI layers. | Container fixtures output ruby version, gem list, native extension paths, server configs; determinism maintained. | + +## Ruby Observation & Runtime (Sprint 54) +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-RUBY-28-008 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-002 | Produce AOC-compliant observations: entrypoints, packages, modules, edges (require/autoload), routes, jobs, tasks, capabilities, configs, warnings. | Observation JSON for fixtures deterministic; edges include reason/form; capability/route/task inventories present; passes AOC lint. | +| SCANNER-ANALYZERS-RUBY-28-009 | TODO | Ruby Analyzer Guild, QA Guild | SCANNER-ANALYZERS-RUBY-28-008 | Fixture suite + performance benchmarks (Rails, Rack, Sinatra, Sidekiq, legacy, .gem, container) with golden outputs. | Fixture set under `fixtures/lang/ruby/ep`; determinism & perf (<4.5s 40k files) CI guard active. | +| SCANNER-ANALYZERS-RUBY-28-010 | TODO | Ruby Analyzer Guild, Signals Guild | SCANNER-ANALYZERS-RUBY-28-008 | Optional runtime evidence integration (if provided logs/metrics) with path hashing, without altering static precedence. | Runtime harness logs merge cleanly with static graph; hashed paths ensure privacy; documentation updated. | +| SCANNER-ANALYZERS-RUBY-28-011 | TODO | Ruby Analyzer Guild, DevOps Guild | SCANNER-ANALYZERS-RUBY-28-008 | Package analyzer plug-in, add CLI (`stella ruby inspect|resolve`) commands, update Offline Kit docs, ensure Worker integration. | Plugin manifest deployed; CLI commands documented/tested; Offline Kit instructions updated; worker restart verified. | +| SCANNER-ANALYZERS-RUBY-28-012 | TODO | Ruby Analyzer Guild | SCANNER-ANALYZERS-RUBY-28-005 | Policy signal emitter: rubygems drift, native extension flags, dangerous constructs counts, TLS verify posture, dynamic require eval warnings. | Policy signal section emitted and validated against fixtures; schema documented; sample policy evaluation added. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj index f6bb6ab9..593ea4ab 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj @@ -1,46 +1,46 @@ - - - net10.0 - preview - enable - enable - true - false - Exe - - - - - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + false + Exe + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj b/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj index f8879698..b7be5c8a 100644 --- a/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj @@ -1,15 +1,15 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj index f8879698..b7be5c8a 100644 --- a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj @@ -1,15 +1,15 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj b/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj index e773101b..4fed40b9 100644 --- a/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj b/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj index 3a6cd8ce..389cfd5f 100644 --- a/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj @@ -1,28 +1,28 @@ - - - net10.0 - preview - enable - enable - true - false - - - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj b/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj index 8cc43a0e..71ce2220 100644 --- a/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj +++ b/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj b/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj index afe325bb..525d94aa 100644 --- a/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj +++ b/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj @@ -1,25 +1,25 @@ - - - net10.0 - enable - enable - false - - - - - - - - - - - - - - - - - - - + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj b/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj index 46556ddc..eff5024a 100644 --- a/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj +++ b/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj @@ -1,19 +1,19 @@ - - - net10.0 - enable - enable - false - - - - - - - - - - - - - + + + net10.0 + enable + enable + false + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs index e7d191bd..8abf64b7 100644 --- a/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs +++ b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs @@ -18,27 +18,32 @@ public sealed class ComponentGraphBuilderTests Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/package.json")), Dependencies = ImmutableArray.Create("pkg:npm/x"), Usage = ComponentUsage.Create(false), - Metadata = new ComponentMetadata - { - Scope = "runtime", - }, - } - }); - - var layer2 = LayerComponentFragment.Create("sha256:layer2", new[] + Metadata = new ComponentMetadata + { + Scope = "runtime", + BuildId = "ABCDEF1234567890ABCDEF1234567890ABCDEF12", + }, + } + }); + + var layer2 = LayerComponentFragment.Create("sha256:layer2", new[] { new ComponentRecord { Identity = ComponentIdentity.Create("pkg:npm/a", "a", "1.0.0"), - LayerDigest = "sha256:layer2", - Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/index.js")), - Dependencies = ImmutableArray.Create("pkg:npm/y"), - Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), - }, - new ComponentRecord - { - Identity = ComponentIdentity.Create("pkg:npm/b", "b", "2.0.0"), - LayerDigest = "sha256:layer2", + LayerDigest = "sha256:layer2", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/index.js")), + Dependencies = ImmutableArray.Create("pkg:npm/y"), + Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), + Metadata = new ComponentMetadata + { + BuildId = "abcdef1234567890abcdef1234567890abcdef12", + }, + }, + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/b", "b", "2.0.0"), + LayerDigest = "sha256:layer2", Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/b/package.json")), } }); @@ -54,9 +59,10 @@ public sealed class ComponentGraphBuilderTests Assert.Equal(new[] { "sha256:layer1", "sha256:layer2" }, componentA.LayerDigests); Assert.True(componentA.Usage.UsedByEntrypoint); Assert.Contains("/app/start.sh", componentA.Usage.Entrypoints); - Assert.Equal(new[] { "pkg:npm/x", "pkg:npm/y" }, componentA.Dependencies); - Assert.Equal("runtime", componentA.Metadata?.Scope); - Assert.Equal(2, componentA.Evidence.Length); + Assert.Equal(new[] { "pkg:npm/x", "pkg:npm/y" }, componentA.Dependencies); + Assert.Equal("runtime", componentA.Metadata?.Scope); + Assert.Equal("abcdef1234567890abcdef1234567890abcdef12", componentA.Metadata?.BuildId); + Assert.Equal(2, componentA.Evidence.Length); var componentB = graph.ComponentMap["pkg:npm/b"]; Assert.Equal("sha256:layer2", componentB.FirstLayerDigest); diff --git a/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs index 48144fb3..1fad0828 100644 --- a/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs +++ b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs @@ -62,24 +62,26 @@ public sealed class ComponentModelsTests Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/package.json")), Dependencies = ImmutableArray.Create("pkg:npm/dep"), Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), - Metadata = new ComponentMetadata - { - Scope = "runtime", - Licenses = new[] { "MIT" }, - Properties = new Dictionary - { - ["source"] = "package-lock.json", - }, - }, - }; - - var json = JsonSerializer.Serialize(record, ScannerJsonOptions.Default); - var deserialized = JsonSerializer.Deserialize(json, ScannerJsonOptions.Default); + Metadata = new ComponentMetadata + { + Scope = "runtime", + Licenses = new[] { "MIT" }, + Properties = new Dictionary + { + ["source"] = "package-lock.json", + }, + BuildId = "5f0c7c3cb4d9f8a4f1c1d5c6b7e8f90123456789", + }, + }; + + var json = JsonSerializer.Serialize(record, ScannerJsonOptions.Default); + var deserialized = JsonSerializer.Deserialize(json, ScannerJsonOptions.Default); Assert.NotNull(deserialized); Assert.Equal(record.Identity.Key, deserialized!.Identity.Key); - Assert.Equal(record.Metadata?.Scope, deserialized.Metadata?.Scope); - Assert.True(deserialized.Usage.UsedByEntrypoint); - Assert.Equal(record.Usage.Entrypoints.AsSpan(), deserialized.Usage.Entrypoints.AsSpan()); - } -} + Assert.Equal(record.Metadata?.Scope, deserialized.Metadata?.Scope); + Assert.Equal(record.Metadata?.BuildId, deserialized.Metadata?.BuildId); + Assert.True(deserialized.Usage.UsedByEntrypoint); + Assert.Equal(record.Usage.Entrypoints.AsSpan(), deserialized.Usage.Entrypoints.AsSpan()); + } +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs b/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs index 29b5eb78..6dec00a0 100644 --- a/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs +++ b/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs @@ -97,23 +97,24 @@ public static class ComponentGraphBuilder _identity = identity; } - public void Include(ComponentRecord record, string layerDigest) - { - _layers.Add(layerDigest); - _dependencies.UnionWith(record.Dependencies); - - foreach (var evidence in record.Evidence) + public void Include(ComponentRecord record, string layerDigest) + { + _layers.Add(layerDigest); + _dependencies.UnionWith(record.Dependencies); + + foreach (var evidence in record.Evidence) { var key = new EvidenceKey(evidence.Kind, evidence.Value, evidence.Source); _evidence[key] = evidence; - } - - if (record.Metadata is not null) - { - _metadata = _metadata is null - ? record.Metadata - : MergeMetadata(_metadata, record.Metadata); - } + } + + if (record.Metadata is not null) + { + var normalized = NormalizeMetadata(record.Metadata); + _metadata = _metadata is null + ? normalized + : MergeMetadata(_metadata, normalized); + } if (record.Usage.UsedByEntrypoint || _usage.UsedByEntrypoint) { @@ -163,20 +164,77 @@ public static class ComponentGraphBuilder } } - private static ComponentMetadata MergeMetadata(ComponentMetadata existing, ComponentMetadata incoming) - { - var scope = existing.Scope ?? incoming.Scope; - - var licenses = MergeLists(existing.Licenses, incoming.Licenses); - var properties = MergeDictionary(existing.Properties, incoming.Properties); - - return new ComponentMetadata - { - Scope = scope, - Licenses = licenses, - Properties = properties, - }; - } + private static ComponentMetadata MergeMetadata(ComponentMetadata existing, ComponentMetadata incoming) + { + var normalizedExisting = NormalizeMetadata(existing); + var normalizedIncoming = NormalizeMetadata(incoming); + + var scope = normalizedExisting.Scope ?? normalizedIncoming.Scope; + + var licenses = MergeLists(normalizedExisting.Licenses, normalizedIncoming.Licenses); + var properties = MergeDictionary(normalizedExisting.Properties, normalizedIncoming.Properties); + var buildId = MergeBuildId(normalizedExisting.BuildId, normalizedIncoming.BuildId); + + return new ComponentMetadata + { + Scope = scope, + Licenses = licenses, + BuildId = buildId, + Properties = properties, + }; + } + + private static ComponentMetadata NormalizeMetadata(ComponentMetadata metadata) + { + var normalizedBuildId = NormalizeBuildId(metadata.BuildId); + if (string.Equals(metadata.BuildId, normalizedBuildId, StringComparison.Ordinal)) + { + return metadata; + } + + return metadata with { BuildId = normalizedBuildId }; + } + + private static string? MergeBuildId(string? left, string? right) + { + if (left is null && right is null) + { + return null; + } + + if (left is null) + { + return right; + } + + if (right is null) + { + return left; + } + + if (string.Equals(left, right, StringComparison.Ordinal)) + { + return left; + } + + return string.CompareOrdinal(left, right) <= 0 ? left : right; + } + + private static string? NormalizeBuildId(string? buildId) + { + if (string.IsNullOrWhiteSpace(buildId)) + { + return null; + } + + var trimmed = buildId.Trim(); + if (trimmed.Length == 0) + { + return null; + } + + return trimmed.ToLowerInvariant(); + } private static IReadOnlyList? MergeLists(IReadOnlyList? left, IReadOnlyList? right) { diff --git a/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs b/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs index 23d14a83..30c527e7 100644 --- a/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs +++ b/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs @@ -87,23 +87,28 @@ public sealed record ComponentEvidence /// /// Optional metadata describing dependency relationships or classification. /// -public sealed record ComponentMetadata -{ - [JsonPropertyName("scope")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Scope { get; init; } - = null; - - [JsonPropertyName("licenses")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public IReadOnlyList? Licenses { get; init; } - = null; - - [JsonPropertyName("properties")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public IReadOnlyDictionary? Properties { get; init; } - = null; -} +public sealed record ComponentMetadata +{ + [JsonPropertyName("scope")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Scope { get; init; } + = null; + + [JsonPropertyName("licenses")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IReadOnlyList? Licenses { get; init; } + = null; + + [JsonPropertyName("buildId")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BuildId { get; init; } + = null; + + [JsonPropertyName("properties")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IReadOnlyDictionary? Properties { get; init; } + = null; +} /// /// Represents a single component discovered within a layer fragment. diff --git a/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj b/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj index 50818b33..a3c89a04 100644 --- a/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj +++ b/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs b/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs index 843ef021..2c705558 100644 --- a/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs +++ b/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs @@ -1,8 +1,9 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Globalization; using StellaOps.Scanner.Core.Contracts; using StellaOps.Scanner.Diff; using StellaOps.Scanner.Core.Utility; @@ -116,8 +117,9 @@ public sealed class ComponentDifferTests var json = DiffJsonSerializer.Serialize(document); using var parsed = JsonDocument.Parse(json); var root = parsed.RootElement; - Assert.Equal("inventory", root.GetProperty("view").GetString()); - Assert.Equal("2025-10-19T10:00:00.000000Z", root.GetProperty("generatedAt").GetString()); + Assert.Equal("inventory", root.GetProperty("view").GetString()); + var generatedAt = DateTimeOffset.Parse(root.GetProperty("generatedAt").GetString()!, CultureInfo.InvariantCulture); + Assert.Equal(request.GeneratedAt, generatedAt); Assert.Equal("sha256:old", root.GetProperty("oldImageDigest").GetString()); Assert.Equal("sha256:new", root.GetProperty("newImageDigest").GetString()); @@ -217,11 +219,11 @@ public sealed class ComponentDifferTests } [Fact] - public void Compute_MetadataChange_WhenEvidenceDiffers() - { - var oldFragments = new[] - { - LayerComponentFragment.Create("sha256:underlay", new[] + public void Compute_MetadataChange_WhenEvidenceDiffers() + { + var oldFragments = new[] + { + LayerComponentFragment.Create("sha256:underlay", new[] { CreateComponent( "pkg:npm/a", @@ -268,28 +270,98 @@ public sealed class ComponentDifferTests Assert.Equal("sha256:overlay", layer.LayerDigest); var change = Assert.Single(layer.Changes); - Assert.Equal(ComponentChangeKind.MetadataChanged, change.Kind); - Assert.Equal("sha256:overlay", change.IntroducingLayer); - Assert.Equal("sha256:underlay", change.RemovingLayer); - Assert.Equal(2, change.NewComponent!.Evidence.Length); - Assert.Equal(1, change.OldComponent!.Evidence.Length); - } + Assert.Equal(ComponentChangeKind.MetadataChanged, change.Kind); + Assert.Equal("sha256:overlay", change.IntroducingLayer); + Assert.Equal("sha256:underlay", change.RemovingLayer); + Assert.Equal(2, change.NewComponent!.Evidence.Length); + Assert.Equal(1, change.OldComponent!.Evidence.Length); + } + + [Fact] + public void Compute_MetadataChange_WhenBuildIdDiffers() + { + var oldFragments = new[] + { + LayerComponentFragment.Create("sha256:base", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.0.0", + layer: "sha256:base", + scope: "runtime", + buildId: "ABCDEF1234567890ABCDEF1234567890ABCDEF12"), + }), + }; + + var newFragments = new[] + { + LayerComponentFragment.Create("sha256:overlay", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.0.0", + layer: "sha256:overlay", + scope: "runtime", + buildId: "6e0d8f6aa1b2c3d4e5f60718293a4b5c6d7e8f90"), + }), + }; + + var request = new ComponentDiffRequest + { + OldGraph = ComponentGraphBuilder.Build(oldFragments), + NewGraph = ComponentGraphBuilder.Build(newFragments), + GeneratedAt = new DateTimeOffset(2025, 10, 19, 13, 0, 0, TimeSpan.Zero), + }; + + var differ = new ComponentDiffer(); + var document = differ.Compute(request); + + Assert.Equal(0, document.Summary.Added); + Assert.Equal(0, document.Summary.Removed); + Assert.Equal(0, document.Summary.VersionChanged); + Assert.Equal(1, document.Summary.MetadataChanged); + + var layer = Assert.Single(document.Layers); + Assert.Equal("sha256:overlay", layer.LayerDigest); + + var change = Assert.Single(layer.Changes); + Assert.Equal(ComponentChangeKind.MetadataChanged, change.Kind); + Assert.Equal("sha256:overlay", change.IntroducingLayer); + Assert.Equal("sha256:base", change.RemovingLayer); + Assert.Equal("abcdef1234567890abcdef1234567890abcdef12", change.OldComponent!.Metadata!.BuildId); + Assert.Equal("6e0d8f6aa1b2c3d4e5f60718293a4b5c6d7e8f90", change.NewComponent!.Metadata!.BuildId); + + var json = DiffJsonSerializer.Serialize(document); + using var parsed = JsonDocument.Parse(json); + var changeJson = parsed.RootElement + .GetProperty("layers")[0] + .GetProperty("changes")[0]; + Assert.Equal("abcdef1234567890abcdef1234567890abcdef12", changeJson.GetProperty("oldComponent").GetProperty("metadata").GetProperty("buildId").GetString()); + Assert.Equal("6e0d8f6aa1b2c3d4e5f60718293a4b5c6d7e8f90", changeJson.GetProperty("newComponent").GetProperty("metadata").GetProperty("buildId").GetString()); + } - private static ComponentRecord CreateComponent( - string key, - string version, - string layer, - ComponentUsage? usage = null, - string? scope = null, - IEnumerable? evidence = null) - { - return new ComponentRecord - { - Identity = ComponentIdentity.Create(key, key.Split('/', 2)[^1], version, purl: key, componentType: "library"), - LayerDigest = layer, - Usage = usage ?? ComponentUsage.Unused, - Metadata = scope is null ? null : new ComponentMetadata { Scope = scope }, - Evidence = evidence is null ? ImmutableArray.Empty : ImmutableArray.CreateRange(evidence), - }; - } -} + private static ComponentRecord CreateComponent( + string key, + string version, + string layer, + ComponentUsage? usage = null, + string? scope = null, + IEnumerable? evidence = null, + string? buildId = null) + { + return new ComponentRecord + { + Identity = ComponentIdentity.Create(key, key.Split('/', 2)[^1], version, purl: key, componentType: "library"), + LayerDigest = layer, + Usage = usage ?? ComponentUsage.Unused, + Metadata = scope is null && buildId is null + ? null + : new ComponentMetadata + { + Scope = scope, + BuildId = buildId, + }, + Evidence = evidence is null ? ImmutableArray.Empty : ImmutableArray.CreateRange(evidence), + }; + } +} diff --git a/src/StellaOps.Scanner.Diff/ComponentDiffer.cs b/src/StellaOps.Scanner.Diff/ComponentDiffer.cs index a4777a24..107635ce 100644 --- a/src/StellaOps.Scanner.Diff/ComponentDiffer.cs +++ b/src/StellaOps.Scanner.Diff/ComponentDiffer.cs @@ -199,16 +199,21 @@ public sealed class ComponentDiffer return false; } - if (!SequenceEqual(left.Licenses, right.Licenses)) - { - return false; - } - - if (!DictionaryEqual(left.Properties, right.Properties)) - { - return false; - } - + if (!SequenceEqual(left.Licenses, right.Licenses)) + { + return false; + } + + if (!string.Equals(NormalizeBuildId(left.BuildId), NormalizeBuildId(right.BuildId), StringComparison.Ordinal)) + { + return false; + } + + if (!DictionaryEqual(left.Properties, right.Properties)) + { + return false; + } + return true; } @@ -270,14 +275,30 @@ public sealed class ComponentDiffer } } - return true; - } - - private static Dictionary ToDictionary(ImmutableArray components) - { - var dictionary = new Dictionary(components.Length, Ordinal); - foreach (var component in components) - { + return true; + } + + private static string? NormalizeBuildId(string? buildId) + { + if (string.IsNullOrWhiteSpace(buildId)) + { + return null; + } + + var trimmed = buildId.Trim(); + if (trimmed.Length == 0) + { + return null; + } + + return trimmed.ToLowerInvariant(); + } + + private static Dictionary ToDictionary(ImmutableArray components) + { + var dictionary = new Dictionary(components.Length, Ordinal); + foreach (var component in components) + { dictionary[component.Identity.Key] = component; } diff --git a/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs b/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs index 460d1c42..775c2b32 100644 --- a/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs +++ b/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs @@ -33,6 +33,9 @@ public sealed class CycloneDxComposerTests ValidateJson(result.Inventory.JsonBytes, expectedComponentCount: 2, expectedView: "inventory"); ValidateJson(result.Usage.JsonBytes, expectedComponentCount: 1, expectedView: "usage"); + var inventoryComponentA = result.Inventory.Components.Single(component => string.Equals(component.Identity.Key, "pkg:npm/a", StringComparison.Ordinal)); + Assert.Equal("abcdef1234567890abcdef1234567890abcdef12", inventoryComponentA.Metadata?.BuildId); + using var inventoryDoc = JsonDocument.Parse(result.Inventory.JsonBytes); var inventoryRoot = inventoryDoc.RootElement; Assert.True(inventoryRoot.TryGetProperty("vulnerabilities", out var inventoryVulnerabilities)); @@ -96,19 +99,20 @@ public sealed class CycloneDxComposerTests Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/package.json")), Dependencies = ImmutableArray.Create("pkg:npm/b"), Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), - Metadata = new ComponentMetadata - { - Scope = "runtime", - Licenses = new[] { "MIT" }, - Properties = new Dictionary - { - ["stellaops:source"] = "package-lock.json", - ["stellaops.os.analyzer"] = "apk", - ["stellaops.os.architecture"] = "x86_64", - }, - }, - } - }), + Metadata = new ComponentMetadata + { + Scope = "runtime", + Licenses = new[] { "MIT" }, + Properties = new Dictionary + { + ["stellaops:source"] = "package-lock.json", + ["stellaops.os.analyzer"] = "apk", + ["stellaops.os.architecture"] = "x86_64", + }, + BuildId = "ABCDEF1234567890ABCDEF1234567890ABCDEF12", + }, + } + }), LayerComponentFragment.Create("sha256:layer2", new[] { new ComponentRecord @@ -209,8 +213,9 @@ public sealed class CycloneDxComposerTests element => element.GetProperty("name").GetString()!, element => element.GetProperty("value").GetString()!, StringComparer.Ordinal); - - Assert.Equal("apk", firstComponentProperties["stellaops.os.analyzer"]); - Assert.Equal("x86_64", firstComponentProperties["stellaops.os.architecture"]); - } -} + + Assert.Equal("apk", firstComponentProperties["stellaops.os.analyzer"]); + Assert.Equal("x86_64", firstComponentProperties["stellaops.os.architecture"]); + Assert.Equal("abcdef1234567890abcdef1234567890abcdef12", firstComponentProperties["stellaops:buildId"]); + } +} diff --git a/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs b/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs index 577d29b1..0761a7b6 100644 --- a/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs +++ b/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs @@ -280,23 +280,32 @@ public sealed class CycloneDxComposer { var properties = new List(); - if (component.Metadata?.Properties is not null) - { - foreach (var property in component.Metadata.Properties.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) - { - properties.Add(new Property + if (component.Metadata?.Properties is not null) + { + foreach (var property in component.Metadata.Properties.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + properties.Add(new Property { Name = property.Key, Value = property.Value, }); - } - } - - properties.Add(new Property { Name = "stellaops:firstLayerDigest", Value = component.FirstLayerDigest }); - if (component.LastLayerDigest is not null) - { - properties.Add(new Property { Name = "stellaops:lastLayerDigest", Value = component.LastLayerDigest }); - } + } + } + + if (!string.IsNullOrWhiteSpace(component.Metadata?.BuildId)) + { + properties.Add(new Property + { + Name = "stellaops:buildId", + Value = component.Metadata!.BuildId, + }); + } + + properties.Add(new Property { Name = "stellaops:firstLayerDigest", Value = component.FirstLayerDigest }); + if (component.LastLayerDigest is not null) + { + properties.Add(new Property { Name = "stellaops:lastLayerDigest", Value = component.LastLayerDigest }); + } if (!component.LayerDigests.IsDefaultOrEmpty) { diff --git a/src/StellaOps.Scanner.Emit/TASKS.md b/src/StellaOps.Scanner.Emit/TASKS.md index 7f23d275..3dfbb050 100644 --- a/src/StellaOps.Scanner.Emit/TASKS.md +++ b/src/StellaOps.Scanner.Emit/TASKS.md @@ -8,5 +8,5 @@ | SCANNER-EMIT-10-604 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-602 | Package artifacts for export + attestation (naming, compression, manifests). | Export pipeline produces deterministic file paths/hashes; integration test with storage passes. | | SCANNER-EMIT-10-605 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-603 | Emit BOM-Index sidecar schema/fixtures (`bom-index@1`) and note CRITICAL PATH for Scheduler. | Schema + fixtures in docs/artifacts/bom-index; tests `BOMIndexGoldenIsStable` green. | | SCANNER-EMIT-10-606 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-605 | Integrate EntryTrace usage flags into BOM-Index; document semantics. | Usage bits present in sidecar; integration tests with EntryTrace fixtures pass. | -| SCANNER-EMIT-17-701 | DOING (2025-10-23) | Emit Guild, Native Analyzer Guild | SCANNER-EMIT-10-602 | Record GNU build-id for ELF components and surface it in inventory/usage SBOM plus diff payloads with deterministic ordering. | Native analyzer emits buildId for every ELF executable/library, SBOM/diff fixtures updated with canonical `buildId` field, regression tests prove stability, docs call out debug-symbol lookup flow. | +| SCANNER-EMIT-17-701 | DONE (2025-10-26) | Emit Guild, Native Analyzer Guild | SCANNER-EMIT-10-602 | Record GNU build-id for ELF components and surface it in inventory/usage SBOM plus diff payloads with deterministic ordering. | Native analyzer emits buildId for every ELF executable/library, SBOM/diff fixtures updated with canonical `buildId` field, regression tests prove stability, docs call out debug-symbol lookup flow. | | SCANNER-EMIT-10-607 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-604, POLICY-CORE-09-005 | Embed scoring inputs, confidence band, and `quietedBy` provenance into CycloneDX 1.6 and DSSE predicates; verify deterministic serialization. | SBOM/attestation fixtures include score, inputs, configVersion, quiet metadata; golden tests confirm canonical output. | diff --git a/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj b/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj index f3cb0889..b2c49e6f 100644 --- a/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj +++ b/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.EntryTrace/TASKS.md b/src/StellaOps.Scanner.EntryTrace/TASKS.md index bd445fa9..b3c65d7d 100644 --- a/src/StellaOps.Scanner.EntryTrace/TASKS.md +++ b/src/StellaOps.Scanner.EntryTrace/TASKS.md @@ -6,11 +6,15 @@ | SCANNER-ENTRYTRACE-10-402 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | Resolve commands across layered rootfs, tracking evidence per hop (PATH hit, layer origin, shebang). | Resolver returns terminal program path with layer attribution for fixtures; deterministic traversal asserted in `EntryTraceAnalyzerTests.ResolveAsync_IsDeterministic`. | | SCANNER-ENTRYTRACE-10-403 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Follow interpreter wrappers (shell → Python/Node/Java launchers) to terminal target, including module/jar detection. | Interpreter tracer reports correct module/script for language launchers; tests cover Python/Node/Java wrappers. | | SCANNER-ENTRYTRACE-10-404 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Build Python entry analyzer detecting venv shebangs, module invocations, `-m` usage and record usage flag. | Python fixtures produce expected module metadata (`python-module` edge) and diagnostics for missing scripts. | -| SCANNER-ENTRYTRACE-10-405 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Implement Node/Java launcher analyzer capturing script/jar targets including npm lifecycle wrappers. | Node/Java fixtures resolved with evidence chain; `RunParts` coverage ensures child scripts traced. | -| SCANNER-ENTRYTRACE-10-406 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Surface explainability + diagnostics for unresolved constructs and emit metrics counters. | Diagnostics catalog enumerates unknown reasons; metrics wired via `EntryTraceMetrics`; explainability doc updated. | -| SCANNER-ENTRYTRACE-10-407 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401..406 | Package EntryTrace analyzers as restart-time plug-ins with manifest + host registration. | Plug-in manifest under `plugins/scanner/entrytrace/`; restart-only policy documented; DI extension exposes `AddEntryTraceAnalyzer`. | - -## Status Review — 2025-10-19 - -- Confirmed Wave 0 instructions for EntryTrace Guild; SCANNER-ENTRYTRACE-10-401..407 already marked complete. -- No outstanding prerequisites identified during review; readiness noted for any follow-on work. +| SCANNER-ENTRYTRACE-10-405 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Implement Node/Java launcher analyzer capturing script/jar targets including npm lifecycle wrappers. | Node/Java fixtures resolved with evidence chain; `RunParts` coverage ensures child scripts traced. | +| SCANNER-ENTRYTRACE-10-406 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Surface explainability + diagnostics for unresolved constructs and emit metrics counters. | Diagnostics catalog enumerates unknown reasons; metrics wired via `EntryTraceMetrics`; explainability doc updated. | +| SCANNER-ENTRYTRACE-10-407 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401..406 | Package EntryTrace analyzers as restart-time plug-ins with manifest + host registration. | Plug-in manifest under `plugins/scanner/entrytrace/`; restart-only policy documented; DI extension exposes `AddEntryTraceAnalyzer`. | +| SCANNER-ENTRYTRACE-18-501 | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-407 | Build OCI config reader and layered rootfs adapter so EntryTrace can hydrate PATH, WorkingDir, User, and provenance from real images. | Fixtures covering tar/dir inputs produce deterministic `IRootFileSystem` descriptors (whiteouts, symlinks, shebangs) and `EntrypointSpecification` derived from config merges with default PATH fallbacks. | +| SCANNER-ENTRYTRACE-18-502 | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-18-501 | Expand chain walker with init shim/user-switch/supervisor recognition plus env/workdir accumulation and guarded edges. | Graph nodes annotate tini/dumb-init/gosu/su-exec/s6/supervisord/runit branches with capability tags, environment deltas, and guard metadata validated against fixture scripts. | +| SCANNER-ENTRYTRACE-18-503 | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-18-502 | Introduce target classifier + EntryPlan handoff with confidence scoring for ELF/Java/.NET/Node/Python and user/workdir context. | Analyzer returns typed targets with confidence metrics and per-branch EntryPlans exercised via golden fixtures and language analyzer stubs. | +| SCANNER-ENTRYTRACE-18-504 | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-18-503 | Emit EntryTrace AOC NDJSON (`entrytrace.entry/node/edge/target/warning/capability`) and wire CLI/service streaming outputs. | NDJSON writer passes determinism tests, CLI/service endpoints stream ordered observations, and diagnostics integrate new warning codes for dynamic eval/glob limits/windows shims. | + +## Status Review — 2025-10-19 + +- Confirmed Wave 0 instructions for EntryTrace Guild; SCANNER-ENTRYTRACE-10-401..407 already marked complete. +- No outstanding prerequisites identified during review; readiness noted for any follow-on work. diff --git a/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj b/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj index d58c88d0..3a90aefb 100644 --- a/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj +++ b/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj @@ -1,21 +1,21 @@ - - - net10.0 - enable - enable - false - - - - - - - - - - - - - - - + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj b/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj index 4ca2ca14..eb780ae7 100644 --- a/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj +++ b/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs b/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs index 97f403eb..df06acaa 100644 --- a/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs +++ b/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs @@ -1,10 +1,14 @@ -using System; -using System.IO; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; -using StellaOps.Notify.Models; -using StellaOps.Scanner.WebService.Contracts; +using System; +using System.IO; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Serialization; +using Xunit.Sdk; namespace StellaOps.Scanner.WebService.Tests; @@ -17,55 +21,128 @@ public sealed class PlatformEventSamplesTests }; [Theory] - [InlineData("scanner.report.ready@1.sample.json", NotifyEventKinds.ScannerReportReady)] - [InlineData("scanner.scan.completed@1.sample.json", NotifyEventKinds.ScannerScanCompleted)] - public void PlatformEventSamplesStayCanonical(string fileName, string expectedKind) - { - var json = LoadSample(fileName); - var notifyEvent = JsonSerializer.Deserialize(json, SerializerOptions); - - Assert.NotNull(notifyEvent); - Assert.Equal(expectedKind, notifyEvent!.Kind); - Assert.NotEqual(Guid.Empty, notifyEvent.EventId); - Assert.NotNull(notifyEvent.Payload); - - AssertCanonical(json, notifyEvent); - AssertReportConsistency(notifyEvent.Payload); - } - - private static void AssertCanonical(string originalJson, NotifyEvent notifyEvent) - { - var canonicalJson = NotifyCanonicalJsonSerializer.Serialize(notifyEvent); - var originalNode = JsonNode.Parse(originalJson) ?? throw new InvalidOperationException("Sample JSON must not be null."); - var canonicalNode = JsonNode.Parse(canonicalJson) ?? throw new InvalidOperationException("Canonical JSON must not be null."); - - Assert.True(JsonNode.DeepEquals(originalNode, canonicalNode), "Platform event sample must remain canonical."); - } - - private static void AssertReportConsistency(JsonNode? payloadNode) - { - var payload = Assert.IsType(payloadNode); - - var reportNode = Assert.IsType(payload["report"]); - var report = reportNode.Deserialize(SerializerOptions); - Assert.NotNull(report); - - var dsseNode = Assert.IsType(payload["dsse"]); - var payloadValueNode = Assert.IsAssignableFrom(dsseNode["payload"]); - var base64Payload = payloadValueNode.GetValue(); - - var canonicalReportBytes = JsonSerializer.SerializeToUtf8Bytes(report!, SerializerOptions); - var expectedPayload = Convert.ToBase64String(canonicalReportBytes); - Assert.Equal(expectedPayload, base64Payload); - - var reportIdReference = Assert.IsAssignableFrom(payload["reportId"]).GetValue(); - Assert.Equal(report!.ReportId, reportIdReference); - } - - private static string LoadSample(string fileName) - { - var path = Path.Combine(AppContext.BaseDirectory, fileName); - Assert.True(File.Exists(path), $"Sample file not found at '{path}'."); - return File.ReadAllText(path); + [InlineData("scanner.event.report.ready@1.sample.json", OrchestratorEventKinds.ScannerReportReady)] + [InlineData("scanner.event.scan.completed@1.sample.json", OrchestratorEventKinds.ScannerScanCompleted)] + public void PlatformEventSamplesStayCanonical(string fileName, string expectedKind) + { + var json = LoadSample(fileName); + var orchestratorEvent = DeserializeOrchestratorEvent(json, expectedKind); + + Assert.NotNull(orchestratorEvent); + Assert.Equal(expectedKind, orchestratorEvent.Kind); + Assert.Equal(1, orchestratorEvent.Version); + Assert.NotEqual(Guid.Empty, orchestratorEvent.EventId); + Assert.NotNull(orchestratorEvent.Payload); + + AssertCanonical(json, orchestratorEvent); + AssertReportConsistency(orchestratorEvent); + } + + private static void AssertCanonical(string originalJson, OrchestratorEvent orchestratorEvent) + { + var canonicalJson = OrchestratorEventSerializer.Serialize(orchestratorEvent); + var originalNode = JsonNode.Parse(originalJson) ?? throw new InvalidOperationException("Sample JSON must not be null."); + var canonicalNode = JsonNode.Parse(canonicalJson) ?? throw new InvalidOperationException("Canonical JSON must not be null."); + + if (!JsonNode.DeepEquals(originalNode, canonicalNode)) + { + throw new Xunit.Sdk.XunitException($"Platform event sample must remain canonical.\nOriginal: {originalJson}\nCanonical: {canonicalJson}"); + } + } + + private static void AssertReportConsistency(OrchestratorEvent orchestratorEvent) + { + switch (orchestratorEvent.Payload) + { + case ReportReadyEventPayload ready: + Assert.Equal(ready.ReportId, ready.Report.ReportId); + Assert.Equal(ready.ScanId, ready.Report.ReportId); + AssertDsseMatchesReport(ready.Dsse, ready.Report); + Assert.False(string.IsNullOrWhiteSpace(ready.Links.Ui)); + Assert.False(string.IsNullOrWhiteSpace(ready.Links.Report)); + Assert.False(string.IsNullOrWhiteSpace(ready.Links.Attestation)); + break; + case ScanCompletedEventPayload completed: + Assert.Equal(completed.ReportId, completed.Report.ReportId); + Assert.Equal(completed.ScanId, completed.Report.ReportId); + AssertDsseMatchesReport(completed.Dsse, completed.Report); + Assert.NotEmpty(completed.Findings); + Assert.False(string.IsNullOrWhiteSpace(completed.Links.Ui)); + Assert.False(string.IsNullOrWhiteSpace(completed.Links.Report)); + Assert.False(string.IsNullOrWhiteSpace(completed.Links.Attestation)); + break; + default: + throw new InvalidOperationException($"Unexpected payload type {orchestratorEvent.Payload.GetType().Name}."); + } + } + + private static void AssertDsseMatchesReport(DsseEnvelopeDto? envelope, ReportDocumentDto report) + { + Assert.NotNull(envelope); + var canonicalReportBytes = JsonSerializer.SerializeToUtf8Bytes(report, SerializerOptions); + var expectedPayload = Convert.ToBase64String(canonicalReportBytes); + Assert.Equal(expectedPayload, envelope.Payload); + } + + private static OrchestratorEvent DeserializeOrchestratorEvent(string json, string expectedKind) + { + var root = JsonNode.Parse(json)?.AsObject() ?? throw new InvalidOperationException("Sample JSON must not be null."); + + var attributes = root["attributes"] is JsonObject attrObj && attrObj.Count > 0 + ? attrObj.ToImmutableDictionary( + pair => pair.Key, + pair => pair.Value?.GetValue() ?? string.Empty, + StringComparer.Ordinal).ToImmutableSortedDictionary(StringComparer.Ordinal) + : null; + + OrchestratorEventScope? scope = null; + if (root["scope"] is JsonObject scopeObj) + { + scope = new OrchestratorEventScope + { + Namespace = scopeObj["namespace"]?.GetValue(), + Repo = scopeObj["repo"]?.GetValue() ?? string.Empty, + Digest = scopeObj["digest"]?.GetValue() ?? string.Empty, + Component = scopeObj["component"]?.GetValue(), + Image = scopeObj["image"]?.GetValue() + }; + } + + var payloadNode = root["payload"] ?? throw new InvalidOperationException("Payload node missing."); + OrchestratorEventPayload payload = expectedKind switch + { + OrchestratorEventKinds.ScannerReportReady => payloadNode.Deserialize(SerializerOptions) + ?? throw new InvalidOperationException("Unable to deserialize report ready payload."), + OrchestratorEventKinds.ScannerScanCompleted => payloadNode.Deserialize(SerializerOptions) + ?? throw new InvalidOperationException("Unable to deserialize scan completed payload."), + _ => throw new InvalidOperationException("Unexpected event kind.") + }; + + return new OrchestratorEvent + { + EventId = Guid.Parse(root["eventId"]!.GetValue()), + Kind = root["kind"]!.GetValue(), + Version = root["version"]?.GetValue() ?? 1, + Tenant = root["tenant"]!.GetValue(), + OccurredAt = DateTimeOffset.Parse(root["occurredAt"]!.GetValue(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal), + RecordedAt = root["recordedAt"] is JsonValue recordedAtNode + ? DateTimeOffset.Parse(recordedAtNode.GetValue(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal) + : null, + Source = root["source"]?.GetValue() ?? string.Empty, + IdempotencyKey = root["idempotencyKey"]?.GetValue() ?? string.Empty, + CorrelationId = root["correlationId"]?.GetValue(), + TraceId = root["traceId"]?.GetValue(), + SpanId = root["spanId"]?.GetValue(), + Scope = scope, + Attributes = attributes, + Payload = payload + }; + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + Assert.True(File.Exists(path), $"Sample file not found at '{path}'."); + return File.ReadAllText(path); } } diff --git a/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs b/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs index c2aa4f03..2f52392a 100644 --- a/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs +++ b/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs @@ -1,20 +1,19 @@ using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Security.Claims; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Auth.Abstractions; -using StellaOps.Notify.Models; -using StellaOps.Policy; -using StellaOps.Scanner.WebService.Contracts; -using StellaOps.Scanner.WebService.Services; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Claims; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Auth.Abstractions; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Services; namespace StellaOps.Scanner.WebService.Tests; @@ -117,40 +116,56 @@ public sealed class ReportEventDispatcherTests await dispatcher.PublishAsync(request, preview, document, envelope, context, cancellationToken); - Assert.Equal(2, publisher.Events.Count); - - var readyEvent = Assert.Single(publisher.Events, evt => evt.Kind == NotifyEventKinds.ScannerReportReady); - Assert.Equal("tenant-alpha", readyEvent.Tenant); - Assert.Equal("api", readyEvent.Scope?.Repo); - Assert.Equal("acme/edge", readyEvent.Scope?.Namespace); - Assert.Equal("sha256:feedface", readyEvent.Scope?.Digest); - Assert.NotNull(readyEvent.Payload); - Assert.Equal("fail", readyEvent.Payload?["verdict"]?.GetValue()); - Assert.Equal("report-abc", readyEvent.Payload?["reportId"]?.GetValue()); - Assert.Equal("signature-value", readyEvent.Payload?["dsse"]?["signatures"]?[0]?["signature"]?.GetValue()); - Assert.Equal(envelope.Payload, readyEvent.Payload?["dsse"]?["payload"]?.GetValue()); - Assert.Equal(1, readyEvent.Payload?["delta"]?["newCritical"]?.GetValue()); - Assert.Equal("CVE-2024-9999", readyEvent.Payload?["delta"]?["kev"]?[0]?.GetValue()); - Assert.Equal("https://scanner.example/ui/reports/report-abc", readyEvent.Payload?["links"]?["ui"]?.GetValue()); - var scanEvent = Assert.Single(publisher.Events, evt => evt.Kind == NotifyEventKinds.ScannerScanCompleted); - Assert.Equal("fail", scanEvent.Payload?["verdict"]?.GetValue()); - Assert.Equal("report-abc", scanEvent.Payload?["reportId"]?.GetValue()); - Assert.Equal("sha256:feedface", scanEvent.Payload?["digest"]?.GetValue()); - Assert.Equal("runtime", scanEvent.Payload?["findings"]?[0]?["reachability"]?.GetValue()); - Assert.Equal("report-abc", scanEvent.Payload?["report"]?["reportId"]?.GetValue()); - Assert.Equal("blocked", scanEvent.Payload?["report"]?["verdict"]?.GetValue()); - Assert.Equal(envelope.Payload, scanEvent.Payload?["dsse"]?["payload"]?.GetValue()); - Assert.Equal(NotifyEventKinds.ScannerScanCompleted, scanEvent.Kind); - } - - private sealed class RecordingEventPublisher : IPlatformEventPublisher - { - public List Events { get; } = new(); - - public Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) - { - Events.Add(@event); - return Task.CompletedTask; + Assert.Equal(2, publisher.Events.Count); + + var readyEvent = Assert.Single(publisher.Events, evt => evt.Kind == OrchestratorEventKinds.ScannerReportReady); + Assert.Equal("tenant-alpha", readyEvent.Tenant); + Assert.Equal("scanner.event.report.ready:tenant-alpha:report-abc", readyEvent.IdempotencyKey); + Assert.Equal("api", readyEvent.Scope?.Repo); + Assert.Equal("acme/edge", readyEvent.Scope?.Namespace); + Assert.Equal("sha256:feedface", readyEvent.Scope?.Digest); + var readyPayload = Assert.IsType(readyEvent.Payload); + Assert.Equal("report-abc", readyPayload.ReportId); + Assert.Equal("report-abc", readyPayload.ScanId); + Assert.Equal("fail", readyPayload.Verdict); + Assert.Equal(0, readyPayload.QuietedFindingCount); + Assert.NotNull(readyPayload.Delta); + Assert.Equal(1, readyPayload.Delta?.NewCritical); + Assert.Contains("CVE-2024-9999", readyPayload.Delta?.Kev ?? Array.Empty()); + Assert.Equal("https://scanner.example/ui/reports/report-abc", readyPayload.Links.Ui); + Assert.Equal("https://scanner.example/api/v1/reports/report-abc", readyPayload.Links.Report); + Assert.Equal("https://scanner.example/api/v1/policy/revisions/rev-42", readyPayload.Links.Policy); + Assert.Equal("https://scanner.example/ui/attestations/report-abc", readyPayload.Links.Attestation); + Assert.Equal(envelope.Payload, readyPayload.Dsse?.Payload); + Assert.Equal("blocked", readyPayload.Report.Verdict); + + var scanEvent = Assert.Single(publisher.Events, evt => evt.Kind == OrchestratorEventKinds.ScannerScanCompleted); + Assert.Equal("tenant-alpha", scanEvent.Tenant); + Assert.Equal("scanner.event.scan.completed:tenant-alpha:report-abc", scanEvent.IdempotencyKey); + Assert.Equal("sha256:feedface", scanEvent.Scope?.Digest); + var scanPayload = Assert.IsType(scanEvent.Payload); + Assert.Equal("report-abc", scanPayload.ReportId); + Assert.Equal("report-abc", scanPayload.ScanId); + Assert.Equal("fail", scanPayload.Verdict); + var finding = Assert.Single(scanPayload.Findings); + Assert.Equal("finding-1", finding.Id); + Assert.Equal("runtime", finding.Reachability); + Assert.Equal("CVE-2024-9999", finding.Cve); + Assert.Equal("https://scanner.example/api/v1/reports/report-abc", scanPayload.Links.Report); + Assert.Equal("https://scanner.example/api/v1/policy/revisions/rev-42", scanPayload.Links.Policy); + Assert.Equal("https://scanner.example/ui/attestations/report-abc", scanPayload.Links.Attestation); + Assert.Equal(envelope.Payload, scanPayload.Dsse?.Payload); + Assert.Equal("blocked", scanPayload.Report.Verdict); + } + + private sealed class RecordingEventPublisher : IPlatformEventPublisher + { + public List Events { get; } = new(); + + public Task PublishAsync(OrchestratorEvent @event, CancellationToken cancellationToken = default) + { + Events.Add(@event); + return Task.CompletedTask; } } } diff --git a/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs index aae3182e..ee922c9e 100644 --- a/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs +++ b/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs @@ -2,7 +2,6 @@ using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; @@ -10,7 +9,6 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using StellaOps.Policy; -using StellaOps.Notify.Models; using StellaOps.Scanner.WebService.Contracts; using StellaOps.Scanner.WebService.Services; using System.Linq; @@ -173,9 +171,10 @@ rules: }); var store = factory.Services.GetRequiredService(); - await store.SaveAsync( + var saveResult = await store.SaveAsync( new PolicySnapshotContent(policyYaml, PolicyDocumentFormat.Yaml, "tester", "seed", "initial"), CancellationToken.None); + var revisionId = saveResult.Snapshot?.RevisionId ?? string.Empty; var recorder = factory.Services.GetRequiredService(); @@ -200,36 +199,49 @@ rules: response.EnsureSuccessStatusCode(); Assert.Equal(2, recorder.Events.Count); - var ready = recorder.Events.Single(evt => evt.Kind == NotifyEventKinds.ScannerReportReady); - var completed = recorder.Events.Single(evt => evt.Kind == NotifyEventKinds.ScannerScanCompleted); + var ready = recorder.Events.Single(evt => evt.Kind == OrchestratorEventKinds.ScannerReportReady); + var completed = recorder.Events.Single(evt => evt.Kind == OrchestratorEventKinds.ScannerScanCompleted); Assert.Equal("default", ready.Tenant); Assert.Equal("default", completed.Tenant); Assert.NotEqual(Guid.Empty, ready.EventId); Assert.NotEqual(Guid.Empty, completed.EventId); - Assert.Equal(ready.Payload?["reportId"]?.GetValue(), completed.Payload?["reportId"]?.GetValue()); - Assert.Equal("sha256:cafebabe", completed.Payload?["digest"]?.GetValue()); - var readyPayload = Assert.IsType(ready.Payload); - Assert.Equal("fail", readyPayload["verdict"]?.GetValue()); + var readyPayload = Assert.IsType(ready.Payload); + var completedPayload = Assert.IsType(completed.Payload); + Assert.Equal(readyPayload.ReportId, completedPayload.ReportId); + Assert.Equal("sha256:cafebabe", completedPayload.ImageDigest); + + Assert.Equal("fail", readyPayload.Verdict); Assert.Equal("acme/edge", ready.Scope?.Namespace); Assert.Equal("api", ready.Scope?.Repo); Assert.Equal("sha256:cafebabe", ready.Scope?.Digest); - Assert.True(readyPayload.ContainsKey("dsse")); - Assert.True(readyPayload.ContainsKey("report")); + Assert.NotNull(readyPayload.Dsse); + Assert.Equal(readyPayload.ReportId, readyPayload.Report.ReportId); + Assert.Equal("http://localhost/ui/reports/" + readyPayload.ReportId, readyPayload.Links.Ui); + Assert.Equal("http://localhost/api/v1/reports/" + readyPayload.ReportId, readyPayload.Links.Report); + if (!string.IsNullOrWhiteSpace(revisionId)) + { + Assert.Equal("http://localhost/api/v1/policy/revisions/" + revisionId, readyPayload.Links.Policy); + } + Assert.Equal("http://localhost/ui/attestations/" + readyPayload.ReportId, readyPayload.Links.Attestation); - var completedPayload = Assert.IsType(completed.Payload); - Assert.Equal("fail", completedPayload["verdict"]?.GetValue()); - Assert.True(completedPayload["findings"] is JsonArray { Count: > 0 }); - - Assert.Equal(2, recorder.Events.Count); + Assert.Equal("fail", completedPayload.Verdict); + Assert.NotEmpty(completedPayload.Findings); + Assert.Equal("finding-42", completedPayload.Findings[0].Id); + Assert.Equal("http://localhost/api/v1/reports/" + completedPayload.ReportId, completedPayload.Links.Report); + if (!string.IsNullOrWhiteSpace(revisionId)) + { + Assert.Equal("http://localhost/api/v1/policy/revisions/" + revisionId, completedPayload.Links.Policy); + } + Assert.Equal("http://localhost/ui/attestations/" + completedPayload.ReportId, completedPayload.Links.Attestation); } private sealed class RecordingPlatformEventPublisher : IPlatformEventPublisher { - public List Events { get; } = new(); + public List Events { get; } = new(); - public Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) + public Task PublishAsync(OrchestratorEvent @event, CancellationToken cancellationToken = default) { Events.Add(@event); return Task.CompletedTask; diff --git a/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj b/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj index 0d940a3c..e60f64d2 100644 --- a/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj +++ b/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj @@ -10,11 +10,11 @@ - + Always - - - Always - - - + + + Always + + + diff --git a/src/StellaOps.Scanner.WebService/Contracts/OrchestratorEventContracts.cs b/src/StellaOps.Scanner.WebService/Contracts/OrchestratorEventContracts.cs new file mode 100644 index 00000000..2bbb500d --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/OrchestratorEventContracts.cs @@ -0,0 +1,277 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.WebService.Contracts; + +internal static class OrchestratorEventKinds +{ + public const string ScannerReportReady = "scanner.event.report.ready"; + public const string ScannerScanCompleted = "scanner.event.scan.completed"; +} + +internal sealed record OrchestratorEvent +{ + [JsonPropertyName("eventId")] + [JsonPropertyOrder(0)] + public Guid EventId { get; init; } + + [JsonPropertyName("kind")] + [JsonPropertyOrder(1)] + public string Kind { get; init; } = string.Empty; + + [JsonPropertyName("version")] + [JsonPropertyOrder(2)] + public int Version { get; init; } = 1; + + [JsonPropertyName("tenant")] + [JsonPropertyOrder(3)] + public string Tenant { get; init; } = string.Empty; + + [JsonPropertyName("occurredAt")] + [JsonPropertyOrder(4)] + public DateTimeOffset OccurredAt { get; init; } + + [JsonPropertyName("recordedAt")] + [JsonPropertyOrder(5)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? RecordedAt { get; init; } + + [JsonPropertyName("source")] + [JsonPropertyOrder(6)] + public string Source { get; init; } = string.Empty; + + [JsonPropertyName("idempotencyKey")] + [JsonPropertyOrder(7)] + public string IdempotencyKey { get; init; } = string.Empty; + + [JsonPropertyName("correlationId")] + [JsonPropertyOrder(8)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CorrelationId { get; init; } + + [JsonPropertyName("traceId")] + [JsonPropertyOrder(9)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TraceId { get; init; } + + [JsonPropertyName("spanId")] + [JsonPropertyOrder(10)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SpanId { get; init; } + + [JsonPropertyName("scope")] + [JsonPropertyOrder(11)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public OrchestratorEventScope? Scope { get; init; } + + [JsonPropertyName("payload")] + [JsonPropertyOrder(12)] + public OrchestratorEventPayload Payload { get; init; } = default!; + + [JsonPropertyName("attributes")] + [JsonPropertyOrder(13)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ImmutableSortedDictionary? Attributes { get; init; } +} + +internal sealed record OrchestratorEventScope +{ + [JsonPropertyName("namespace")] + [JsonPropertyOrder(0)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Namespace { get; init; } + + [JsonPropertyName("repo")] + [JsonPropertyOrder(1)] + public string Repo { get; init; } = string.Empty; + + [JsonPropertyName("digest")] + [JsonPropertyOrder(2)] + public string Digest { get; init; } = string.Empty; + + [JsonPropertyName("component")] + [JsonPropertyOrder(3)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Component { get; init; } + + [JsonPropertyName("image")] + [JsonPropertyOrder(4)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Image { get; init; } +} + +internal abstract record OrchestratorEventPayload; + +internal sealed record ReportReadyEventPayload : OrchestratorEventPayload +{ + [JsonPropertyName("reportId")] + [JsonPropertyOrder(0)] + public string ReportId { get; init; } = string.Empty; + + [JsonPropertyName("scanId")] + [JsonPropertyOrder(1)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScanId { get; init; } + + [JsonPropertyName("imageDigest")] + [JsonPropertyOrder(2)] + public string ImageDigest { get; init; } = string.Empty; + + [JsonPropertyName("generatedAt")] + [JsonPropertyOrder(3)] + public DateTimeOffset GeneratedAt { get; init; } + + [JsonPropertyName("verdict")] + [JsonPropertyOrder(4)] + public string Verdict { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + [JsonPropertyOrder(5)] + public ReportSummaryDto Summary { get; init; } = new(); + + [JsonPropertyName("delta")] + [JsonPropertyOrder(6)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ReportDeltaPayload? Delta { get; init; } + + [JsonPropertyName("quietedFindingCount")] + [JsonPropertyOrder(7)] + public int QuietedFindingCount { get; init; } + + [JsonPropertyName("policy")] + [JsonPropertyOrder(8)] + public ReportPolicyDto Policy { get; init; } = new(); + + [JsonPropertyName("links")] + [JsonPropertyOrder(9)] + public ReportLinksPayload Links { get; init; } = new(); + + [JsonPropertyName("dsse")] + [JsonPropertyOrder(10)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DsseEnvelopeDto? Dsse { get; init; } + + [JsonPropertyName("report")] + [JsonPropertyOrder(11)] + public ReportDocumentDto Report { get; init; } = new(); +} + +internal sealed record ScanCompletedEventPayload : OrchestratorEventPayload +{ + [JsonPropertyName("reportId")] + [JsonPropertyOrder(0)] + public string ReportId { get; init; } = string.Empty; + + [JsonPropertyName("scanId")] + [JsonPropertyOrder(1)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScanId { get; init; } + + [JsonPropertyName("imageDigest")] + [JsonPropertyOrder(2)] + public string ImageDigest { get; init; } = string.Empty; + + [JsonPropertyName("verdict")] + [JsonPropertyOrder(3)] + public string Verdict { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + [JsonPropertyOrder(4)] + public ReportSummaryDto Summary { get; init; } = new(); + + [JsonPropertyName("delta")] + [JsonPropertyOrder(5)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ReportDeltaPayload? Delta { get; init; } + + [JsonPropertyName("policy")] + [JsonPropertyOrder(6)] + public ReportPolicyDto Policy { get; init; } = new(); + + [JsonPropertyName("findings")] + [JsonPropertyOrder(7)] + public IReadOnlyList Findings { get; init; } = Array.Empty(); + + [JsonPropertyName("links")] + [JsonPropertyOrder(8)] + public ReportLinksPayload Links { get; init; } = new(); + + [JsonPropertyName("dsse")] + [JsonPropertyOrder(9)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DsseEnvelopeDto? Dsse { get; init; } + + [JsonPropertyName("report")] + [JsonPropertyOrder(10)] + public ReportDocumentDto Report { get; init; } = new(); +} + +internal sealed record ReportDeltaPayload +{ + [JsonPropertyName("newCritical")] + [JsonPropertyOrder(0)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? NewCritical { get; init; } + + [JsonPropertyName("newHigh")] + [JsonPropertyOrder(1)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? NewHigh { get; init; } + + [JsonPropertyName("kev")] + [JsonPropertyOrder(2)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IReadOnlyList? Kev { get; init; } +} + +internal sealed record ReportLinksPayload +{ + [JsonPropertyName("ui")] + [JsonPropertyOrder(0)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Ui { get; init; } + + [JsonPropertyName("report")] + [JsonPropertyOrder(1)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Report { get; init; } + + [JsonPropertyName("policy")] + [JsonPropertyOrder(2)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Policy { get; init; } + + [JsonPropertyName("attestation")] + [JsonPropertyOrder(3)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Attestation { get; init; } +} + +internal sealed record FindingSummaryPayload +{ + [JsonPropertyName("id")] + [JsonPropertyOrder(0)] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("severity")] + [JsonPropertyOrder(1)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Severity { get; init; } + + [JsonPropertyName("cve")] + [JsonPropertyOrder(2)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Cve { get; init; } + + [JsonPropertyName("purl")] + [JsonPropertyOrder(3)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Purl { get; init; } + + [JsonPropertyName("reachability")] + [JsonPropertyOrder(4)] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Reachability { get; init; } +} diff --git a/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs b/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs index 5359e73c..0535a2aa 100644 --- a/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs +++ b/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs @@ -12,47 +12,81 @@ internal static class OpenApiRegistrationExtensions ArgumentNullException.ThrowIfNull(services); var extensionType = Type.GetType("Microsoft.Extensions.DependencyInjection.OpenApiServiceCollectionExtensions, Microsoft.AspNetCore.OpenApi"); - if (extensionType is not null) - { - var method = extensionType - .GetMethods(BindingFlags.Public | BindingFlags.Static) - .FirstOrDefault(m => - string.Equals(m.Name, "AddOpenApi", StringComparison.Ordinal) && - m.GetParameters().Length == 2); - - if (method is not null) - { - var result = method.Invoke(null, new object?[] { services, null }); - if (result is IServiceCollection collection) - { - return collection; - } - } - } - - services.AddEndpointsApiExplorer(); - return services; - } + if (extensionType is not null) + { + var method = extensionType + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + string.Equals(m.Name, "AddOpenApi", StringComparison.Ordinal)); + + if (method is not null) + { + try + { + var parameters = method.GetParameters(); + object?[] arguments = parameters.Length switch + { + 2 => new object?[] { services, null }, + 3 => new object?[] { services, "scanner", null }, + _ => Array.Empty() + }; + + if (arguments.Length == parameters.Length) + { + var result = method.Invoke(null, arguments); + if (result is IServiceCollection collection) + { + return collection; + } + } + } + catch + { + // Fall back to minimal explorer registration below. + } + } + } + + services.AddEndpointsApiExplorer(); + return services; + } public static WebApplication MapOpenApiIfAvailable(this WebApplication app) { ArgumentNullException.ThrowIfNull(app); var extensionType = Type.GetType("Microsoft.AspNetCore.Builder.OpenApiApplicationBuilderExtensions, Microsoft.AspNetCore.OpenApi"); - if (extensionType is not null) - { - var method = extensionType - .GetMethods(BindingFlags.Public | BindingFlags.Static) - .FirstOrDefault(m => - string.Equals(m.Name, "MapOpenApi", StringComparison.Ordinal) && - m.GetParameters().Length == 1); - - if (method is not null) - { - method.Invoke(null, new object?[] { app }); - } - } - - return app; - } -} + if (extensionType is not null) + { + var method = extensionType + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + string.Equals(m.Name, "MapOpenApi", StringComparison.Ordinal)); + + if (method is not null) + { + try + { + var parameters = method.GetParameters(); + object?[] arguments = parameters.Length switch + { + 1 => new object?[] { app }, + 2 => new object?[] { app, "scanner" }, + _ => Array.Empty() + }; + + if (arguments.Length == parameters.Length) + { + method.Invoke(null, arguments); + } + } + catch + { + // Ignore failures and continue without OpenAPI mapping. + } + } + } + + return app; + } +} diff --git a/src/StellaOps.Scanner.WebService/Serialization/OrchestratorEventSerializer.cs b/src/StellaOps.Scanner.WebService/Serialization/OrchestratorEventSerializer.cs new file mode 100644 index 00000000..282a2cef --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Serialization/OrchestratorEventSerializer.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Serialization; + +internal static class OrchestratorEventSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + public static string Serialize(OrchestratorEvent @event) + => JsonSerializer.Serialize(@event, CompactOptions); + + public static string SerializeIndented(OrchestratorEvent @event) + => JsonSerializer.Serialize(@event, PrettyOptions); + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = writeIndented, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private static readonly ImmutableDictionary PropertyOrder = new Dictionary + { + [typeof(OrchestratorEvent)] = new[] + { + "eventId", + "kind", + "version", + "tenant", + "occurredAt", + "recordedAt", + "source", + "idempotencyKey", + "correlationId", + "traceId", + "spanId", + "scope", + "payload", + "attributes" + }, + [typeof(OrchestratorEventScope)] = new[] + { + "namespace", + "repo", + "digest", + "component", + "image" + }, + [typeof(ReportReadyEventPayload)] = new[] + { + "reportId", + "scanId", + "imageDigest", + "generatedAt", + "verdict", + "summary", + "delta", + "quietedFindingCount", + "policy", + "links", + "dsse", + "report" + }, + [typeof(ScanCompletedEventPayload)] = new[] + { + "reportId", + "scanId", + "imageDigest", + "verdict", + "summary", + "delta", + "policy", + "findings", + "links", + "dsse", + "report" + }, + [typeof(ReportDeltaPayload)] = new[] + { + "newCritical", + "newHigh", + "kev" + }, + [typeof(ReportLinksPayload)] = new[] + { + "ui", + "report", + "policy", + "attestation" + }, + [typeof(FindingSummaryPayload)] = new[] + { + "id", + "severity", + "cve", + "purl", + "reachability" + }, + [typeof(ReportPolicyDto)] = new[] + { + "revisionId", + "digest" + }, + [typeof(ReportSummaryDto)] = new[] + { + "total", + "blocked", + "warned", + "ignored", + "quieted" + }, + [typeof(ReportDocumentDto)] = new[] + { + "reportId", + "imageDigest", + "generatedAt", + "verdict", + "policy", + "summary", + "verdicts", + "issues" + }, + [typeof(DsseEnvelopeDto)] = new[] + { + "payloadType", + "payload", + "signatures" + }, + [typeof(DsseSignatureDto)] = new[] + { + "keyId", + "algorithm", + "signature" + } + }.ToImmutableDictionary(); + + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options) + ?? throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(property => GetOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int GetOrder(Type type, string propertyName) + { + if (PropertyOrder.TryGetValue(type, out var order) && Array.IndexOf(order, propertyName) is { } index and >= 0) + { + return index; + } + + if (type.BaseType is not null) + { + return GetOrder(type.BaseType, propertyName); + } + + return int.MaxValue; + } + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs index eca26ba3..b1f1c841 100644 --- a/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs +++ b/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs @@ -1,16 +1,16 @@ using System.Threading; using System.Threading.Tasks; -using StellaOps.Notify.Models; - -namespace StellaOps.Scanner.WebService.Services; - -/// -/// Publishes platform events to the internal bus consumed by downstream services (Notify, UI, etc.). -/// -public interface IPlatformEventPublisher -{ - /// - /// Publishes the supplied event envelope. - /// - Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default); -} +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Services; + +/// +/// Publishes orchestrator events to the internal bus consumed by downstream services. +/// +internal interface IPlatformEventPublisher +{ + /// + /// Publishes the supplied event envelope. + /// + Task PublishAsync(OrchestratorEvent @event, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs index a12797a2..d38ce3b0 100644 --- a/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs +++ b/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs @@ -1,7 +1,7 @@ -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Notify.Models; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.WebService.Contracts; namespace StellaOps.Scanner.WebService.Services; @@ -17,18 +17,18 @@ internal sealed class NullPlatformEventPublisher : IPlatformEventPublisher _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) - { - if (@event is null) - { - throw new ArgumentNullException(nameof(@event)); - } - - if (_logger.IsEnabled(LogLevel.Debug)) - { - _logger.LogDebug("Suppressing publish for event {EventKind} (tenant {Tenant}).", @event.Kind, @event.Tenant); - } - - return Task.CompletedTask; - } -} + public Task PublishAsync(OrchestratorEvent @event, CancellationToken cancellationToken = default) + { + if (@event is null) + { + throw new ArgumentNullException(nameof(@event)); + } + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("Suppressing publish for orchestrator event {EventKind} (tenant {Tenant}).", @event.Kind, @event.Tenant); + } + + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs index 22e98e2f..8482a193 100644 --- a/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs +++ b/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs @@ -1,12 +1,12 @@ using System; -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StackExchange.Redis; -using StellaOps.Notify.Models; -using StellaOps.Scanner.WebService.Options; +using StackExchange.Redis; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Scanner.WebService.Serialization; namespace StellaOps.Scanner.WebService.Services; @@ -49,21 +49,22 @@ internal sealed class RedisPlatformEventPublisher : IPlatformEventPublisher, IAs _maxStreamLength = _options.MaxStreamLength > 0 ? _options.MaxStreamLength : null; } - public async Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) - { - ArgumentNullException.ThrowIfNull(@event); - cancellationToken.ThrowIfCancellationRequested(); - - var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); - var payload = NotifyCanonicalJsonSerializer.Serialize(@event); - - var entries = new NameValueEntry[] - { - new("event", payload), - new("kind", @event.Kind), - new("tenant", @event.Tenant), - new("ts", @event.Ts.ToString("O")) - }; + public async Task PublishAsync(OrchestratorEvent @event, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(@event); + cancellationToken.ThrowIfCancellationRequested(); + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + var payload = OrchestratorEventSerializer.Serialize(@event); + + var entries = new NameValueEntry[] + { + new("event", payload), + new("kind", @event.Kind), + new("tenant", @event.Tenant), + new("occurredAt", @event.OccurredAt.ToString("O")), + new("idempotencyKey", @event.IdempotencyKey) + }; int? maxLength = null; if (_maxStreamLength.HasValue) diff --git a/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs b/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs index 7ad4f8b4..71f48534 100644 --- a/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs +++ b/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs @@ -1,520 +1,583 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Security.Claims; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.Json.Serialization; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using StellaOps.Auth.Abstractions; -using StellaOps.Notify.Models; -using StellaOps.Policy; -using StellaOps.Scanner.WebService.Contracts; - -namespace StellaOps.Scanner.WebService.Services; - -internal sealed class ReportEventDispatcher : IReportEventDispatcher -{ - private const string DefaultTenant = "default"; - private const string Actor = "scanner.webservice"; - - private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - - private readonly IPlatformEventPublisher _publisher; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - - public ReportEventDispatcher( - IPlatformEventPublisher publisher, - TimeProvider timeProvider, - ILogger logger) - { - _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); - _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task PublishAsync( - ReportRequestDto request, - PolicyPreviewResponse preview, - ReportDocumentDto document, - DsseEnvelopeDto? envelope, - HttpContext httpContext, - CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(request); - ArgumentNullException.ThrowIfNull(preview); - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(httpContext); - - cancellationToken.ThrowIfCancellationRequested(); - - var now = _timeProvider.GetUtcNow(); - var tenant = ResolveTenant(httpContext); - var scope = BuildScope(request, document); - var attributes = BuildAttributes(document); - - var reportPayload = BuildReportReadyPayload(request, preview, document, envelope, httpContext); - var reportEvent = NotifyEvent.Create( - eventId: Guid.NewGuid(), - kind: NotifyEventKinds.ScannerReportReady, - tenant: tenant, - ts: document.GeneratedAt == default ? now : document.GeneratedAt, - payload: reportPayload, - scope: scope, - actor: Actor, - attributes: attributes); - - await PublishSafelyAsync(reportEvent, document.ReportId, cancellationToken).ConfigureAwait(false); - - var scanPayload = BuildScanCompletedPayload(request, preview, document, envelope); - var scanEvent = NotifyEvent.Create( - eventId: Guid.NewGuid(), - kind: NotifyEventKinds.ScannerScanCompleted, - tenant: tenant, - ts: document.GeneratedAt == default ? now : document.GeneratedAt, - payload: scanPayload, - scope: scope, - actor: Actor, - attributes: attributes); - - await PublishSafelyAsync(scanEvent, document.ReportId, cancellationToken).ConfigureAwait(false); - } - - private async Task PublishSafelyAsync(NotifyEvent @event, string reportId, CancellationToken cancellationToken) - { - try - { - await _publisher.PublishAsync(@event, cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - throw; - } - catch (Exception ex) - { - _logger.LogError( - ex, - "Failed to publish event {EventKind} for report {ReportId}.", - @event.Kind, - reportId); - } - } - - private static string ResolveTenant(HttpContext context) - { - var tenant = context.User?.FindFirstValue(StellaOpsClaimTypes.Tenant); - if (!string.IsNullOrWhiteSpace(tenant)) - { - return tenant.Trim(); - } - - if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerTenant)) - { - var headerValue = headerTenant.ToString(); - if (!string.IsNullOrWhiteSpace(headerValue)) - { - return headerValue.Trim(); - } - } - - return DefaultTenant; - } - - private static NotifyEventScope BuildScope(ReportRequestDto request, ReportDocumentDto document) - { - var repository = ResolveRepository(request); - var (ns, repo) = SplitRepository(repository); - - var digest = string.IsNullOrWhiteSpace(document.ImageDigest) - ? request.ImageDigest ?? string.Empty - : document.ImageDigest; - - return NotifyEventScope.Create( - @namespace: ns, - repo: string.IsNullOrWhiteSpace(repo) ? "(unknown)" : repo, - digest: string.IsNullOrWhiteSpace(digest) ? "(unknown)" : digest); - } - - private static string ResolveRepository(ReportRequestDto request) - { - if (request.Findings is { Count: > 0 }) - { - foreach (var finding in request.Findings) - { - if (!string.IsNullOrWhiteSpace(finding.Repository)) - { - return finding.Repository!.Trim(); - } - - if (!string.IsNullOrWhiteSpace(finding.Image)) - { - return finding.Image!.Trim(); - } - } - } - - return string.Empty; - } - - private static (string? Namespace, string Repo) SplitRepository(string repository) - { - if (string.IsNullOrWhiteSpace(repository)) - { - return (null, string.Empty); - } - - var normalized = repository.Trim(); - var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (segments.Length == 0) - { - return (null, normalized); - } - - if (segments.Length == 1) - { - return (null, segments[0]); - } - - var repo = segments[^1]; - var ns = string.Join('/', segments[..^1]); - return (ns, repo); - } - - private static IEnumerable> BuildAttributes(ReportDocumentDto document) - { - var attributes = new List>(capacity: 4) - { - new("reportId", document.ReportId) - }; - - if (!string.IsNullOrWhiteSpace(document.Policy.RevisionId)) - { - attributes.Add(new("policyRevisionId", document.Policy.RevisionId!)); - } - - if (!string.IsNullOrWhiteSpace(document.Policy.Digest)) - { - attributes.Add(new("policyDigest", document.Policy.Digest!)); - } - - attributes.Add(new("verdict", document.Verdict)); - return attributes; - } - - private static JsonObject BuildReportReadyPayload( - ReportRequestDto request, - PolicyPreviewResponse preview, - ReportDocumentDto document, - DsseEnvelopeDto? envelope, - HttpContext context) - { - var payload = new JsonObject - { - ["reportId"] = document.ReportId, - ["generatedAt"] = document.GeneratedAt == default - ? null - : JsonValue.Create(document.GeneratedAt), - ["verdict"] = MapVerdict(document.Verdict), - ["summary"] = JsonSerializer.SerializeToNode(document.Summary, JsonOptions), - ["delta"] = BuildDelta(preview, request), - ["links"] = BuildLinks(context, document), - ["quietedFindingCount"] = document.Summary.Quieted - }; - - payload.RemoveNulls(); - - if (envelope is not null) - { - payload["dsse"] = JsonSerializer.SerializeToNode(envelope, JsonOptions); - } - - payload["report"] = JsonSerializer.SerializeToNode(document, JsonOptions); - return payload; - } - - private static JsonObject BuildScanCompletedPayload( - ReportRequestDto request, - PolicyPreviewResponse preview, - ReportDocumentDto document, - DsseEnvelopeDto? envelope) - { - var payload = new JsonObject - { - ["reportId"] = document.ReportId, - ["digest"] = document.ImageDigest, - ["summary"] = JsonSerializer.SerializeToNode(document.Summary, JsonOptions), - ["verdict"] = MapVerdict(document.Verdict), - ["policy"] = JsonSerializer.SerializeToNode(document.Policy, JsonOptions), - ["delta"] = BuildDelta(preview, request), - ["report"] = JsonSerializer.SerializeToNode(document, JsonOptions) - }; - - if (envelope is not null) - { - payload["dsse"] = JsonSerializer.SerializeToNode(envelope, JsonOptions); - } - - payload["findings"] = BuildFindingSummaries(request); - payload.RemoveNulls(); - return payload; - } - - private static JsonArray BuildFindingSummaries(ReportRequestDto request) - { - var array = new JsonArray(); - if (request.Findings is { Count: > 0 }) - { - foreach (var finding in request.Findings) - { - if (string.IsNullOrWhiteSpace(finding.Id)) - { - continue; - } - - var summary = new JsonObject - { - ["id"] = finding.Id, - ["severity"] = finding.Severity, - ["cve"] = finding.Cve, - ["purl"] = finding.Purl, - ["reachability"] = ResolveReachability(finding.Tags) - }; - - summary.RemoveNulls(); - array.Add(summary); - } - } - - return array; - } - - private static string? ResolveReachability(IReadOnlyList? tags) - { - if (tags is null) - { - return null; - } - - foreach (var tag in tags) - { - if (string.IsNullOrWhiteSpace(tag)) - { - continue; - } - - if (tag.StartsWith("reachability:", StringComparison.OrdinalIgnoreCase)) - { - return tag["reachability:".Length..]; - } - } - - return null; - } - - private static JsonObject BuildDelta(PolicyPreviewResponse preview, ReportRequestDto request) - { - var delta = new JsonObject(); - if (preview.Diffs.IsDefaultOrEmpty) - { - return delta; - } - - var findings = BuildFindingsIndex(request.Findings); - var kevIds = new SortedSet(StringComparer.OrdinalIgnoreCase); - var newCritical = 0; - var newHigh = 0; - - foreach (var diff in preview.Diffs) - { - var projected = diff.Projected; - if (projected is null || string.IsNullOrWhiteSpace(projected.FindingId)) - { - continue; - } - - if (!findings.TryGetValue(projected.FindingId, out var finding)) - { - finding = null; - } - - if (IsNewlyImportant(diff)) - { - var severity = finding?.Severity; - if (string.Equals(severity, "Critical", StringComparison.OrdinalIgnoreCase)) - { - newCritical++; - } - else if (string.Equals(severity, "High", StringComparison.OrdinalIgnoreCase)) - { - newHigh++; - } - - var kevId = ResolveKevIdentifier(finding); - if (!string.IsNullOrWhiteSpace(kevId)) - { - kevIds.Add(kevId); - } - } - } - - if (newCritical > 0) - { - delta["newCritical"] = newCritical; - } - - if (newHigh > 0) - { - delta["newHigh"] = newHigh; - } - - if (kevIds.Count > 0) - { - var kev = new JsonArray(); - foreach (var id in kevIds) - { - kev.Add(id); - } - - delta["kev"] = kev; - } - - return delta; - } - - private static ImmutableDictionary BuildFindingsIndex( - IReadOnlyList? findings) - { - if (findings is null || findings.Count == 0) - { - return ImmutableDictionary.Empty; - } - - var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - foreach (var finding in findings) - { - if (string.IsNullOrWhiteSpace(finding.Id)) - { - continue; - } - - if (!builder.ContainsKey(finding.Id)) - { - builder.Add(finding.Id, finding); - } - } - - return builder.ToImmutable(); - } - - private static bool IsNewlyImportant(PolicyVerdictDiff diff) - { - var projected = diff.Projected.Status; - var baseline = diff.Baseline.Status; - - return projected switch - { - PolicyVerdictStatus.Blocked or PolicyVerdictStatus.Escalated - => baseline != PolicyVerdictStatus.Blocked && baseline != PolicyVerdictStatus.Escalated, - PolicyVerdictStatus.Warned or PolicyVerdictStatus.Deferred or PolicyVerdictStatus.RequiresVex - => baseline != PolicyVerdictStatus.Warned - && baseline != PolicyVerdictStatus.Deferred - && baseline != PolicyVerdictStatus.RequiresVex - && baseline != PolicyVerdictStatus.Blocked - && baseline != PolicyVerdictStatus.Escalated, - _ => false - }; - } - - private static string? ResolveKevIdentifier(PolicyPreviewFindingDto? finding) - { - if (finding is null) - { - return null; - } - - var tags = finding.Tags; - if (tags is not null) - { - foreach (var tag in tags) - { - if (string.IsNullOrWhiteSpace(tag)) - { - continue; - } - - if (string.Equals(tag, "kev", StringComparison.OrdinalIgnoreCase)) - { - return finding.Cve; - } - - if (tag.StartsWith("kev:", StringComparison.OrdinalIgnoreCase)) - { - var value = tag["kev:".Length..]; - if (!string.IsNullOrWhiteSpace(value)) - { - return value.Trim(); - } - } - } - } - - return finding.Cve; - } - - private static JsonObject BuildLinks(HttpContext context, ReportDocumentDto document) - { - var links = new JsonObject(); - - if (context.Request.Host.HasValue) - { - var scheme = string.IsNullOrWhiteSpace(context.Request.Scheme) ? "https" : context.Request.Scheme; - var builder = new UriBuilder(scheme, context.Request.Host.Host) - { - Port = context.Request.Host.Port ?? -1, - Path = $"/ui/reports/{Uri.EscapeDataString(document.ReportId)}" - }; - links["ui"] = builder.Uri.ToString(); - } - - return links; - } - - private static string MapVerdict(string verdict) - => verdict.ToLowerInvariant() switch - { - "blocked" or "fail" => "fail", - "escalated" => "fail", - "warn" or "warned" or "deferred" or "requiresvex" => "warn", - _ => "pass" - }; -} - -internal static class ReportEventDispatcherExtensions -{ - public static void RemoveNulls(this JsonObject jsonObject) - { - if (jsonObject is null) - { - return; - } - - var keysToRemove = new List(); - foreach (var pair in jsonObject) - { - if (pair.Value is null || pair.Value.GetValueKind() == JsonValueKind.Null) - { - keysToRemove.Add(pair.Key); - } - } - - foreach (var key in keysToRemove) - { - jsonObject.Remove(key); - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using System.Security.Claims; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Options; + +namespace StellaOps.Scanner.WebService.Services; + +internal sealed class ReportEventDispatcher : IReportEventDispatcher +{ + private const string DefaultTenant = "default"; + private const string Source = "scanner.webservice"; + + private readonly IPlatformEventPublisher _publisher; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly string[] _apiBaseSegments; + private readonly string _reportsSegment; + private readonly string _policySegment; + + public ReportEventDispatcher( + IPlatformEventPublisher publisher, + IOptions options, + TimeProvider timeProvider, + ILogger logger) + { + _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + var apiOptions = options.Value.Api ?? new ScannerWebServiceOptions.ApiOptions(); + _apiBaseSegments = SplitSegments(apiOptions.BasePath); + _reportsSegment = string.IsNullOrWhiteSpace(apiOptions.ReportsSegment) + ? "reports" + : apiOptions.ReportsSegment.Trim('/'); + _policySegment = string.IsNullOrWhiteSpace(apiOptions.PolicySegment) + ? "policy" + : apiOptions.PolicySegment.Trim('/'); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PublishAsync( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope, + HttpContext httpContext, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(preview); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(httpContext); + + cancellationToken.ThrowIfCancellationRequested(); + + var now = _timeProvider.GetUtcNow(); + var occurredAt = document.GeneratedAt == default ? now : document.GeneratedAt; + var tenant = ResolveTenant(httpContext); + var scope = BuildScope(request, document); + var attributes = BuildAttributes(document); + var links = BuildLinks(httpContext, document, envelope); + var correlationId = document.ReportId; + var (traceId, spanId) = ResolveTraceContext(); + + var reportEvent = new OrchestratorEvent + { + EventId = Guid.NewGuid(), + Kind = OrchestratorEventKinds.ScannerReportReady, + Version = 1, + Tenant = tenant, + OccurredAt = occurredAt, + RecordedAt = now, + Source = Source, + IdempotencyKey = BuildIdempotencyKey(OrchestratorEventKinds.ScannerReportReady, tenant, document.ReportId), + CorrelationId = correlationId, + TraceId = traceId, + SpanId = spanId, + Scope = scope, + Attributes = attributes, + Payload = BuildReportReadyPayload(request, preview, document, envelope, links, correlationId) + }; + + await PublishSafelyAsync(reportEvent, document.ReportId, cancellationToken).ConfigureAwait(false); + + var scanCompletedEvent = new OrchestratorEvent + { + EventId = Guid.NewGuid(), + Kind = OrchestratorEventKinds.ScannerScanCompleted, + Version = 1, + Tenant = tenant, + OccurredAt = occurredAt, + RecordedAt = now, + Source = Source, + IdempotencyKey = BuildIdempotencyKey(OrchestratorEventKinds.ScannerScanCompleted, tenant, correlationId), + CorrelationId = correlationId, + TraceId = traceId, + SpanId = spanId, + Scope = scope, + Attributes = attributes, + Payload = BuildScanCompletedPayload(request, preview, document, envelope, links, correlationId) + }; + + await PublishSafelyAsync(scanCompletedEvent, document.ReportId, cancellationToken).ConfigureAwait(false); + } + + private async Task PublishSafelyAsync(OrchestratorEvent @event, string reportId, CancellationToken cancellationToken) + { + try + { + await _publisher.PublishAsync(@event, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to publish orchestrator event {EventKind} for report {ReportId}.", + @event.Kind, + reportId); + } + } + + private static string ResolveTenant(HttpContext context) + { + var tenant = context.User?.FindFirstValue(StellaOpsClaimTypes.Tenant); + if (!string.IsNullOrWhiteSpace(tenant)) + { + return tenant.Trim(); + } + + if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerTenant)) + { + var headerValue = headerTenant.ToString(); + if (!string.IsNullOrWhiteSpace(headerValue)) + { + return headerValue.Trim(); + } + } + + return DefaultTenant; + } + + private static OrchestratorEventScope BuildScope(ReportRequestDto request, ReportDocumentDto document) + { + var repository = ResolveRepository(request); + var (ns, repo) = SplitRepository(repository); + + var digest = string.IsNullOrWhiteSpace(document.ImageDigest) + ? request.ImageDigest ?? string.Empty + : document.ImageDigest; + + return new OrchestratorEventScope + { + Namespace = ns, + Repo = string.IsNullOrWhiteSpace(repo) ? "(unknown)" : repo, + Digest = string.IsNullOrWhiteSpace(digest) ? "(unknown)" : digest + }; + } + + private static ImmutableSortedDictionary BuildAttributes(ReportDocumentDto document) + { + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + builder["reportId"] = document.ReportId; + builder["verdict"] = document.Verdict; + + if (!string.IsNullOrWhiteSpace(document.Policy.RevisionId)) + { + builder["policyRevisionId"] = document.Policy.RevisionId!; + } + + if (!string.IsNullOrWhiteSpace(document.Policy.Digest)) + { + builder["policyDigest"] = document.Policy.Digest!; + } + + return builder.ToImmutable(); + } + + private static ReportReadyEventPayload BuildReportReadyPayload( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope, + ReportLinksPayload links, + string correlationId) + { + return new ReportReadyEventPayload + { + ReportId = document.ReportId, + ScanId = correlationId, + ImageDigest = document.ImageDigest, + GeneratedAt = document.GeneratedAt, + Verdict = MapVerdict(document.Verdict), + Summary = document.Summary, + Delta = BuildDelta(preview, request), + QuietedFindingCount = document.Summary.Quieted, + Policy = document.Policy, + Links = links, + Dsse = envelope, + Report = document + }; + } + + private static ScanCompletedEventPayload BuildScanCompletedPayload( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope, + ReportLinksPayload links, + string correlationId) + { + return new ScanCompletedEventPayload + { + ReportId = document.ReportId, + ScanId = correlationId, + ImageDigest = document.ImageDigest, + Verdict = MapVerdict(document.Verdict), + Summary = document.Summary, + Delta = BuildDelta(preview, request), + Policy = document.Policy, + Findings = BuildFindingSummaries(request), + Links = links, + Dsse = envelope, + Report = document + }; + } + + private ReportLinksPayload BuildLinks(HttpContext context, ReportDocumentDto document, DsseEnvelopeDto? envelope) + { + if (!context.Request.Host.HasValue) + { + return new ReportLinksPayload(); + } + + var uiLink = BuildAbsoluteUri(context, "ui", "reports", document.ReportId); + var reportLink = BuildAbsoluteUri(context, ConcatSegments(_apiBaseSegments, _reportsSegment, document.ReportId)); + var policyLink = string.IsNullOrWhiteSpace(document.Policy.RevisionId) + ? null + : BuildAbsoluteUri(context, ConcatSegments(_apiBaseSegments, _policySegment, "revisions", document.Policy.RevisionId)); + var attestationLink = envelope is null + ? null + : BuildAbsoluteUri(context, "ui", "attestations", document.ReportId); + + return new ReportLinksPayload + { + Ui = uiLink, + Report = reportLink, + Policy = policyLink, + Attestation = attestationLink + }; + } + + private static ReportDeltaPayload? BuildDelta(PolicyPreviewResponse preview, ReportRequestDto request) + { + if (preview.Diffs.IsDefaultOrEmpty) + { + return null; + } + + var findings = BuildFindingsIndex(request.Findings); + var kevIds = new SortedSet(StringComparer.OrdinalIgnoreCase); + var newCritical = 0; + var newHigh = 0; + + foreach (var diff in preview.Diffs) + { + var projected = diff.Projected; + if (projected is null || string.IsNullOrWhiteSpace(projected.FindingId)) + { + continue; + } + + findings.TryGetValue(projected.FindingId, out var finding); + + if (IsNewlyImportant(diff)) + { + var severity = finding?.Severity; + if (string.Equals(severity, "Critical", StringComparison.OrdinalIgnoreCase)) + { + newCritical++; + } + else if (string.Equals(severity, "High", StringComparison.OrdinalIgnoreCase)) + { + newHigh++; + } + + var kevId = ResolveKevIdentifier(finding); + if (!string.IsNullOrWhiteSpace(kevId)) + { + kevIds.Add(kevId); + } + } + } + + if (newCritical == 0 && newHigh == 0 && kevIds.Count == 0) + { + return null; + } + + return new ReportDeltaPayload + { + NewCritical = newCritical > 0 ? newCritical : null, + NewHigh = newHigh > 0 ? newHigh : null, + Kev = kevIds.Count > 0 ? kevIds.ToArray() : null + }; + } + + private static string BuildAbsoluteUri(HttpContext context, params string[] segments) + => BuildAbsoluteUri(context, segments.AsEnumerable()); + + private static string BuildAbsoluteUri(HttpContext context, IEnumerable segments) + { + var normalized = segments + .Where(segment => !string.IsNullOrWhiteSpace(segment)) + .Select(segment => segment.Trim('/')) + .Where(segment => segment.Length > 0) + .ToArray(); + + if (!context.Request.Host.HasValue || normalized.Length == 0) + { + return string.Empty; + } + + var scheme = string.IsNullOrWhiteSpace(context.Request.Scheme) ? "https" : context.Request.Scheme; + var builder = new UriBuilder(scheme, context.Request.Host.Host) + { + Port = context.Request.Host.Port ?? -1, + Path = "/" + string.Join('/', normalized.Select(Uri.EscapeDataString)), + Query = string.Empty, + Fragment = string.Empty + }; + + return builder.Uri.ToString(); + } + + private string[] ConcatSegments(IEnumerable prefix, params string[] suffix) + { + var segments = new List(); + foreach (var segment in prefix) + { + if (!string.IsNullOrWhiteSpace(segment)) + { + segments.Add(segment.Trim('/')); + } + } + + foreach (var segment in suffix) + { + if (!string.IsNullOrWhiteSpace(segment)) + { + segments.Add(segment.Trim('/')); + } + } + + return segments.ToArray(); + } + + private static string[] SplitSegments(string? path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return Array.Empty(); + } + + return path.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + private static ImmutableDictionary BuildFindingsIndex( + IReadOnlyList? findings) + { + if (findings is null || findings.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var finding in findings) + { + if (string.IsNullOrWhiteSpace(finding.Id)) + { + continue; + } + + if (!builder.ContainsKey(finding.Id)) + { + builder.Add(finding.Id, finding); + } + } + + return builder.ToImmutable(); + } + + private static IReadOnlyList BuildFindingSummaries(ReportRequestDto request) + { + if (request.Findings is not { Count: > 0 }) + { + return Array.Empty(); + } + + var summaries = new List(request.Findings.Count); + foreach (var finding in request.Findings) + { + if (string.IsNullOrWhiteSpace(finding.Id)) + { + continue; + } + + summaries.Add(new FindingSummaryPayload + { + Id = finding.Id, + Severity = finding.Severity, + Cve = finding.Cve, + Purl = finding.Purl, + Reachability = ResolveReachability(finding.Tags) + }); + } + + return summaries; + } + + private static string ResolveRepository(ReportRequestDto request) + { + if (request.Findings is { Count: > 0 }) + { + foreach (var finding in request.Findings) + { + if (!string.IsNullOrWhiteSpace(finding.Repository)) + { + return finding.Repository!.Trim(); + } + + if (!string.IsNullOrWhiteSpace(finding.Image)) + { + return finding.Image!.Trim(); + } + } + } + + return string.Empty; + } + + private static (string? Namespace, string Repo) SplitRepository(string repository) + { + if (string.IsNullOrWhiteSpace(repository)) + { + return (null, string.Empty); + } + + var normalized = repository.Trim(); + var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length == 0) + { + return (null, normalized); + } + + if (segments.Length == 1) + { + return (null, segments[0]); + } + + var repo = segments[^1]; + var ns = string.Join('/', segments[..^1]); + return (ns, repo); + } + + private static bool IsNewlyImportant(PolicyVerdictDiff diff) + { + var projected = diff.Projected.Status; + var baseline = diff.Baseline.Status; + + return projected switch + { + PolicyVerdictStatus.Blocked or PolicyVerdictStatus.Escalated + => baseline != PolicyVerdictStatus.Blocked && baseline != PolicyVerdictStatus.Escalated, + PolicyVerdictStatus.Warned or PolicyVerdictStatus.Deferred or PolicyVerdictStatus.RequiresVex + => baseline != PolicyVerdictStatus.Warned + && baseline != PolicyVerdictStatus.Deferred + && baseline != PolicyVerdictStatus.RequiresVex + && baseline != PolicyVerdictStatus.Blocked + && baseline != PolicyVerdictStatus.Escalated, + _ => false + }; + } + + private static string? ResolveKevIdentifier(PolicyPreviewFindingDto? finding) + { + if (finding is null) + { + return null; + } + + var tags = finding.Tags; + if (tags is not null) + { + foreach (var tag in tags) + { + if (string.IsNullOrWhiteSpace(tag)) + { + continue; + } + + if (string.Equals(tag, "kev", StringComparison.OrdinalIgnoreCase)) + { + return finding.Cve; + } + + if (tag.StartsWith("kev:", StringComparison.OrdinalIgnoreCase)) + { + var value = tag["kev:".Length..]; + if (!string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + } + } + + return finding.Cve; + } + + private static string? ResolveReachability(IReadOnlyList? tags) + { + if (tags is null) + { + return null; + } + + foreach (var tag in tags) + { + if (string.IsNullOrWhiteSpace(tag)) + { + continue; + } + + if (tag.StartsWith("reachability:", StringComparison.OrdinalIgnoreCase)) + { + return tag["reachability:".Length..]; + } + } + + return null; + } + + private static string MapVerdict(string verdict) + => verdict.ToLowerInvariant() switch + { + "blocked" or "fail" => "fail", + "escalated" => "fail", + "warn" or "warned" or "deferred" or "requiresvex" => "warn", + _ => "pass" + }; + + private static string BuildIdempotencyKey(string kind, string tenant, string identifier) + => $"{kind}:{tenant}:{identifier}".ToLowerInvariant(); + + private static (string? TraceId, string? SpanId) ResolveTraceContext() + { + var activity = Activity.Current; + if (activity is null) + { + return (null, null); + } + + var traceId = activity.TraceId.ToString(); + var spanId = activity.SpanId.ToString(); + return (traceId, spanId); + } +} diff --git a/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj b/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj index b26d3d04..b67ca90a 100644 --- a/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj +++ b/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - StellaOps.Scanner.WebService - - - - - - + + + net10.0 + preview + enable + enable + true + StellaOps.Scanner.WebService + + + + + + - + diff --git a/src/StellaOps.Scanner.WebService/TASKS.md b/src/StellaOps.Scanner.WebService/TASKS.md index 1d59b9c0..3a5c634e 100644 --- a/src/StellaOps.Scanner.WebService/TASKS.md +++ b/src/StellaOps.Scanner.WebService/TASKS.md @@ -16,7 +16,8 @@ | SCANNER-RUNTIME-12-304 | DONE (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | Surface attestation verification status by integrating Authority/Attestor Rekor validation (beyond presence-only). | `/policy/runtime` maps Rekor UUIDs through the runtime attestation verifier so `rekor.verified` reflects attestor outcomes; webhook/CLI coverage added. | | SCANNER-RUNTIME-12-305 | DONE (2025-10-24) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, SCANNER-RUNTIME-12-302 | Promote shared fixtures with Zastava/CLI and add end-to-end automation for `/runtime/events` + `/policy/runtime`. | Runtime policy integration test + CLI-aligned fixture assert confidence, metadata JSON, and Rekor verification; docs note shared contract. | | SCANNER-EVENTS-15-201 | DONE (2025-10-20) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Emit `scanner.report.ready` and `scanner.scan.completed` events (bus adapters + tests). | Event envelopes published to queue with schemas; fixtures committed; Notify consumption test passes. | -| SCANNER-EVENTS-16-301 | TODO | Scanner WebService Guild | ORCH-SVC-38-101, NOTIFY-SVC-38-001 | Emit orchestrator-compatible envelopes (`scanner.event.*`) and update integration tests to verify Notifier ingestion (no Redis queue coupling). | Tests assert envelope schema + orchestrator publish; Notifier consumer harness passes; docs updated with new event contract. | +| SCANNER-EVENTS-16-301 | BLOCKED (2025-10-26) | Scanner WebService Guild | ORCH-SVC-38-101, NOTIFY-SVC-38-001 | Emit orchestrator-compatible envelopes (`scanner.event.*`) and update integration tests to verify Notifier ingestion (no Redis queue coupling). | Tests assert envelope schema + orchestrator publish; Notifier consumer harness passes; docs updated with new event contract. Blocked by .NET 10 preview OpenAPI/Auth dependency drift preventing `dotnet test` completion. | +| SCANNER-EVENTS-16-302 | DOING (2025-10-26) | Scanner WebService Guild | SCANNER-EVENTS-16-301 | Extend orchestrator event links (report/policy/attestation) once endpoints are finalised across gateway + console. | Links section covers UI/API targets; downstream consumers validated; docs/samples updated. | | SCANNER-RUNTIME-17-401 | DONE (2025-10-25) | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-OBS-17-005, SCANNER-EMIT-17-701, POLICY-RUNTIME-17-201 | Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. | Runtime events store normalized digests + build IDs with supporting indexes, runtime policy responses surface `buildIds`, tests/docs updated, and CLI/API consumers can derive debug-store paths deterministically. | ## Graph Explorer v1 (Sprint 21) @@ -39,3 +40,6 @@ - 2025-10-20: Follow-ups SCANNER-RUNTIME-12-303/304/305 track canonical verdict integration, attestation verification, and cross-guild fixture validation for runtime APIs. - 2025-10-21: Hardened progress streaming determinism by sorting `data` payload keys within `ScanProgressStream`; added regression `ProgressStreamDataKeysAreSortedDeterministically` ensuring JSONL ordering. - 2025-10-24: `/policy/runtime` now streams through PolicyPreviewService + attestation verifier; CLI and webhook fixtures updated alongside Zastava observer batching completion. +- 2025-10-26: SCANNER-EVENTS-16-302 populates orchestrator link payloads (UI, API report lookup, policy revision, attestation) pending cross-service integration; samples/tests updated. +- 2025-10-26: Coordinate with Gateway + Console owners to confirm final API/UX paths for report, policy revision, and attestation links before promoting SCANNER-EVENTS-16-301 out of BLOCKED. +- 2025-10-26: SCANNER-EVENTS-16-301 emitting new orchestrator envelopes; solution-wide `dotnet test` currently blocked by preview `Microsoft.AspNetCore.OpenApi` APIs and missing `StellaOps.Auth` dependency wiring. JSON Schemas validated via `ajv`; service-level verification pending SDK alignment. diff --git a/src/StellaOps.Scheduler.ImpactIndex.Tests/RoaringImpactIndexTests.cs b/src/StellaOps.Scheduler.ImpactIndex.Tests/RoaringImpactIndexTests.cs new file mode 100644 index 00000000..9bea4c9e --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex.Tests/RoaringImpactIndexTests.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Immutable; +using System.IO; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scheduler.ImpactIndex.Ingestion; +using StellaOps.Scheduler.Models; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Emit.Index; +using Xunit; + +namespace StellaOps.Scheduler.ImpactIndex.Tests; + +public sealed class RoaringImpactIndexTests +{ + [Fact] + public async Task IngestAsync_RegistersComponentsAndUsage() + { + var (stream, digest) = CreateBomIndex( + ComponentIdentity.Create("pkg:npm/a@1.0.0", "a", "1.0.0", "pkg:npm/a@1.0.0"), + ComponentUsage.Create(true, new[] { "/app/start.sh" })); + + var index = new RoaringImpactIndex(NullLogger.Instance); + var request = new ImpactIndexIngestionRequest + { + TenantId = "tenant-alpha", + ImageDigest = digest, + Registry = "docker.io", + Repository = "library/alpine", + Namespaces = ImmutableArray.Create("team-a"), + Tags = ImmutableArray.Create("3.20"), + Labels = ImmutableSortedDictionary.CreateRange(StringComparer.OrdinalIgnoreCase, new[] + { + new KeyValuePair("env", "prod") + }), + BomIndexStream = stream, + }; + + await index.IngestAsync(request, CancellationToken.None); + + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + var impactSet = await index.ResolveByPurlsAsync(new[] { "pkg:npm/a@1.0.0" }, usageOnly: false, selector); + + impactSet.Images.Should().HaveCount(1); + impactSet.Images[0].ImageDigest.Should().Be(digest); + impactSet.Images[0].Tags.Should().ContainSingle(tag => tag == "3.20"); + impactSet.Images[0].UsedByEntrypoint.Should().BeTrue(); + + var usageOnly = await index.ResolveByPurlsAsync(new[] { "pkg:npm/a@1.0.0" }, usageOnly: true, selector); + usageOnly.Images.Should().HaveCount(1); + } + + [Fact] + public async Task IngestAsync_ReplacesExistingImageData() + { + var component = ComponentIdentity.Create("pkg:npm/a@1.0.0", "a", "1.0.0", "pkg:npm/a@1.0.0"); + var (initialStream, digest) = CreateBomIndex(component, ComponentUsage.Create(false)); + var index = new RoaringImpactIndex(NullLogger.Instance); + + await index.IngestAsync(new ImpactIndexIngestionRequest + { + TenantId = "tenant-alpha", + ImageDigest = digest, + Registry = "docker.io", + Repository = "library/alpine", + Tags = ImmutableArray.Create("v1"), + BomIndexStream = initialStream, + }); + + var (updatedStream, _) = CreateBomIndex(component, ComponentUsage.Create(true, new[] { "/start.sh" }), digest); + await index.IngestAsync(new ImpactIndexIngestionRequest + { + TenantId = "tenant-alpha", + ImageDigest = digest, + Registry = "docker.io", + Repository = "library/alpine", + Tags = ImmutableArray.Create("v2"), + BomIndexStream = updatedStream, + }); + + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + var impactSet = await index.ResolveByPurlsAsync(new[] { "pkg:npm/a@1.0.0" }, usageOnly: true, selector); + + impactSet.Images.Should().HaveCount(1); + impactSet.Images[0].Tags.Should().ContainSingle(tag => tag == "v2"); + impactSet.Images[0].UsedByEntrypoint.Should().BeTrue(); + } + + [Fact] + public async Task ResolveByPurlsAsync_RespectsTenantNamespaceAndTagFilters() + { + var component = ComponentIdentity.Create("pkg:npm/a@1.0.0", "a", "1.0.0", "pkg:npm/a@1.0.0"); + var (tenantStream, tenantDigest) = CreateBomIndex(component, ComponentUsage.Create(true, new[] { "/start.sh" })); + var (otherStream, otherDigest) = CreateBomIndex(component, ComponentUsage.Create(false)); + + var index = new RoaringImpactIndex(NullLogger.Instance); + + await index.IngestAsync(new ImpactIndexIngestionRequest + { + TenantId = "tenant-alpha", + ImageDigest = tenantDigest, + Registry = "docker.io", + Repository = "library/service", + Namespaces = ImmutableArray.Create("team-alpha"), + Tags = ImmutableArray.Create("prod-eu"), + BomIndexStream = tenantStream, + }); + + await index.IngestAsync(new ImpactIndexIngestionRequest + { + TenantId = "tenant-beta", + ImageDigest = otherDigest, + Registry = "docker.io", + Repository = "library/service", + Namespaces = ImmutableArray.Create("team-beta"), + Tags = ImmutableArray.Create("staging-us"), + BomIndexStream = otherStream, + }); + + var selector = new Selector( + SelectorScope.AllImages, + tenantId: "tenant-alpha", + namespaces: new[] { "team-alpha" }, + includeTags: new[] { "prod-*" }); + + var result = await index.ResolveByPurlsAsync(new[] { "pkg:npm/a@1.0.0" }, usageOnly: true, selector); + + result.Images.Should().ContainSingle(image => image.ImageDigest == tenantDigest); + result.Images[0].Tags.Should().Contain("prod-eu"); + } + + [Fact] + public async Task ResolveAllAsync_UsageOnlyFiltersEntrypointImages() + { + var component = ComponentIdentity.Create("pkg:npm/a@1.0.0", "a", "1.0.0", "pkg:npm/a@1.0.0"); + var (entryStream, entryDigest) = CreateBomIndex(component, ComponentUsage.Create(true, new[] { "/start.sh" })); + var nonEntryDigestValue = "sha256:" + new string('1', 64); + var (nonEntryStream, nonEntryDigest) = CreateBomIndex(component, ComponentUsage.Create(false), nonEntryDigestValue); + + var index = new RoaringImpactIndex(NullLogger.Instance); + + await index.IngestAsync(new ImpactIndexIngestionRequest + { + TenantId = "tenant-alpha", + ImageDigest = entryDigest, + Registry = "docker.io", + Repository = "library/service", + BomIndexStream = entryStream, + }); + + await index.IngestAsync(new ImpactIndexIngestionRequest + { + TenantId = "tenant-alpha", + ImageDigest = nonEntryDigest, + Registry = "docker.io", + Repository = "library/service", + BomIndexStream = nonEntryStream, + }); + + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + + var usageOnly = await index.ResolveAllAsync(selector, usageOnly: true); + usageOnly.Images.Should().ContainSingle(image => image.ImageDigest == entryDigest); + + var allImages = await index.ResolveAllAsync(selector, usageOnly: false); + allImages.Images.Should().HaveCount(2); + } + + private static (Stream Stream, string Digest) CreateBomIndex(ComponentIdentity identity, ComponentUsage usage, string? digest = null) + { + var layer = LayerComponentFragment.Create( + "sha256:layer1", + new[] + { + new ComponentRecord + { + Identity = identity, + LayerDigest = "sha256:layer1", + Usage = usage, + } + }); + + var graph = ComponentGraphBuilder.Build(new[] { layer }); + var effectiveDigest = digest ?? "sha256:" + Guid.NewGuid().ToString("N"); + var builder = new BomIndexBuilder(); + var artifact = builder.Build(new BomIndexBuildRequest + { + ImageDigest = effectiveDigest, + Graph = graph, + GeneratedAt = DateTimeOffset.UtcNow, + }); + + return (new MemoryStream(artifact.Bytes, writable: false), effectiveDigest); + } +} diff --git a/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj b/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj index 30b5f162..5652955b 100644 --- a/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj +++ b/src/StellaOps.Scheduler.ImpactIndex.Tests/StellaOps.Scheduler.ImpactIndex.Tests.csproj @@ -21,5 +21,6 @@ + diff --git a/src/StellaOps.Scheduler.ImpactIndex/ImpactImageRecord.cs b/src/StellaOps.Scheduler.ImpactIndex/ImpactImageRecord.cs new file mode 100644 index 00000000..395ee54e --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/ImpactImageRecord.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Scheduler.ImpactIndex; + +internal sealed record ImpactImageRecord( + int ImageId, + string TenantId, + string Digest, + string Registry, + string Repository, + ImmutableArray Namespaces, + ImmutableArray Tags, + ImmutableSortedDictionary Labels, + DateTimeOffset GeneratedAt, + ImmutableArray Components, + ImmutableArray EntrypointComponents); diff --git a/src/StellaOps.Scheduler.ImpactIndex/Ingestion/BomIndexReader.cs b/src/StellaOps.Scheduler.ImpactIndex/Ingestion/BomIndexReader.cs new file mode 100644 index 00000000..4764196c --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/Ingestion/BomIndexReader.cs @@ -0,0 +1,119 @@ +using System.Buffers.Binary; +using System.Collections.Immutable; +using System.Globalization; +using System.Text; +using Collections.Special; + +namespace StellaOps.Scheduler.ImpactIndex.Ingestion; + +internal sealed record BomIndexComponent(string Key, bool UsedByEntrypoint); + +internal sealed record BomIndexDocument(string ImageDigest, DateTimeOffset GeneratedAt, ImmutableArray Components); + +internal static class BomIndexReader +{ + private const int HeaderMagicLength = 7; + private static readonly byte[] Magic = Encoding.ASCII.GetBytes("BOMIDX1"); + + public static BomIndexDocument Read(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + + using var reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true); + Span magicBuffer = stackalloc byte[HeaderMagicLength]; + if (reader.Read(magicBuffer) != HeaderMagicLength || !magicBuffer.SequenceEqual(Magic)) + { + throw new InvalidOperationException("Invalid BOM index header magic."); + } + + var version = reader.ReadUInt16(); + if (version != 1) + { + throw new NotSupportedException($"Unsupported BOM index version '{version}'."); + } + + var flags = reader.ReadUInt16(); + var hasEntrypoints = (flags & 0x1) == 1; + + var digestLength = reader.ReadUInt16(); + var digestBytes = reader.ReadBytes(digestLength); + var imageDigest = Encoding.UTF8.GetString(digestBytes); + + var generatedAtMicros = reader.ReadInt64(); + var generatedAt = DateTimeOffset.FromUnixTimeMilliseconds(generatedAtMicros / 1000) + .AddTicks((generatedAtMicros % 1000) * TimeSpan.TicksPerMillisecond / 1000); + + var layerCount = checked((int)reader.ReadUInt32()); + var componentCount = checked((int)reader.ReadUInt32()); + var entrypointCount = checked((int)reader.ReadUInt32()); + + // Layer table (we only need to skip entries but validate length) + for (var i = 0; i < layerCount; i++) + { + _ = ReadUtf8String(reader); + } + + var componentKeys = new string[componentCount]; + for (var i = 0; i < componentCount; i++) + { + componentKeys[i] = ReadUtf8String(reader); + } + + for (var i = 0; i < componentCount; i++) + { + var length = reader.ReadUInt32(); + if (length > 0) + { + var payload = reader.ReadBytes(checked((int)length)); + using var bitmapStream = new MemoryStream(payload, writable: false); + _ = RoaringBitmap.Deserialize(bitmapStream); + } + } + + var entrypointPresence = new bool[componentCount]; + if (hasEntrypoints && entrypointCount > 0) + { + // Entrypoint table (skip strings) + for (var i = 0; i < entrypointCount; i++) + { + _ = ReadUtf8String(reader); + } + + for (var i = 0; i < componentCount; i++) + { + var length = reader.ReadUInt32(); + if (length == 0) + { + entrypointPresence[i] = false; + continue; + } + + var payload = reader.ReadBytes(checked((int)length)); + using var bitmapStream = new MemoryStream(payload, writable: false); + var bitmap = RoaringBitmap.Deserialize(bitmapStream); + entrypointPresence[i] = bitmap.Any(); + } + } + + var builder = ImmutableArray.CreateBuilder(componentCount); + for (var i = 0; i < componentCount; i++) + { + var key = componentKeys[i]; + builder.Add(new BomIndexComponent(key, entrypointPresence[i])); + } + + return new BomIndexDocument(imageDigest, generatedAt, builder.MoveToImmutable()); + } + + private static string ReadUtf8String(BinaryReader reader) + { + var length = reader.ReadUInt16(); + if (length == 0) + { + return string.Empty; + } + + var bytes = reader.ReadBytes(length); + return Encoding.UTF8.GetString(bytes); + } +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/Ingestion/ImpactIndexIngestionRequest.cs b/src/StellaOps.Scheduler.ImpactIndex/Ingestion/ImpactIndexIngestionRequest.cs new file mode 100644 index 00000000..3bf02262 --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/Ingestion/ImpactIndexIngestionRequest.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Immutable; +using System.IO; + +namespace StellaOps.Scheduler.ImpactIndex.Ingestion; + +/// +/// Describes a BOM-Index ingestion payload for the scheduler impact index. +/// +public sealed record ImpactIndexIngestionRequest +{ + public required string TenantId { get; init; } + + public required string ImageDigest { get; init; } + + public required string Registry { get; init; } + + public required string Repository { get; init; } + + public ImmutableArray Namespaces { get; init; } = ImmutableArray.Empty; + + public ImmutableArray Tags { get; init; } = ImmutableArray.Empty; + + public ImmutableSortedDictionary Labels { get; init; } = ImmutableSortedDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + + public required Stream BomIndexStream { get; init; } + = Stream.Null; +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/RoaringImpactIndex.cs b/src/StellaOps.Scheduler.ImpactIndex/RoaringImpactIndex.cs new file mode 100644 index 00000000..d1466b0d --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/RoaringImpactIndex.cs @@ -0,0 +1,481 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using Collections.Special; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.ImpactIndex.Ingestion; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.ImpactIndex; + +/// +/// Roaring bitmap-backed implementation of the scheduler impact index. +/// +public sealed class RoaringImpactIndex : IImpactIndex +{ + private readonly object _gate = new(); + + private readonly Dictionary _imageIds = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _images = new(); + private readonly Dictionary _containsByPurl = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _usedByEntrypointByPurl = new(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public RoaringImpactIndex(ILogger logger, TimeProvider? timeProvider = null) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task IngestAsync(ImpactIndexIngestionRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(request.BomIndexStream); + + using var buffer = new MemoryStream(); + await request.BomIndexStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + buffer.Position = 0; + + var document = BomIndexReader.Read(buffer); + if (!string.Equals(document.ImageDigest, request.ImageDigest, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"BOM-Index digest mismatch. Header '{document.ImageDigest}', request '{request.ImageDigest}'."); + } + + var tenantId = request.TenantId ?? throw new ArgumentNullException(nameof(request.TenantId)); + var registry = request.Registry ?? throw new ArgumentNullException(nameof(request.Registry)); + var repository = request.Repository ?? throw new ArgumentNullException(nameof(request.Repository)); + + var namespaces = request.Namespaces.IsDefault ? ImmutableArray.Empty : request.Namespaces; + var tags = request.Tags.IsDefault ? ImmutableArray.Empty : request.Tags; + var labels = request.Labels.Count == 0 + ? ImmutableSortedDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase) + : request.Labels; + + var componentKeys = document.Components + .Select(component => component.Key) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + var entrypointComponents = document.Components + .Where(component => component.UsedByEntrypoint) + .Select(component => component.Key) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + lock (_gate) + { + var imageId = EnsureImageId(request.ImageDigest); + + if (_images.TryGetValue(imageId, out var existing)) + { + RemoveImageComponents(existing); + } + + var metadata = new ImpactImageRecord( + imageId, + tenantId, + request.ImageDigest, + registry, + repository, + namespaces, + tags, + labels, + document.GeneratedAt, + componentKeys, + entrypointComponents); + + _images[imageId] = metadata; + _imageIds[request.ImageDigest] = imageId; + + foreach (var key in componentKeys) + { + var bitmap = _containsByPurl.GetValueOrDefault(key); + _containsByPurl[key] = AddImageToBitmap(bitmap, imageId); + } + + foreach (var key in entrypointComponents) + { + var bitmap = _usedByEntrypointByPurl.GetValueOrDefault(key); + _usedByEntrypointByPurl[key] = AddImageToBitmap(bitmap, imageId); + } + } + + _logger.LogInformation( + "ImpactIndex ingested BOM-Index for {Digest} ({TenantId}/{Repository}). Components={ComponentCount} EntrypointComponents={EntrypointCount}", + request.ImageDigest, + tenantId, + repository, + componentKeys.Length, + entrypointComponents.Length); + } + + public ValueTask ResolveByPurlsAsync( + IEnumerable purls, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(ResolveByPurlsCore(purls, usageOnly, selector)); + + public ValueTask ResolveByVulnerabilitiesAsync( + IEnumerable vulnerabilityIds, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(CreateEmptyImpactSet(selector, usageOnly)); + + public ValueTask ResolveAllAsync( + Selector selector, + bool usageOnly, + CancellationToken cancellationToken = default) + => ValueTask.FromResult(ResolveAllCore(selector, usageOnly)); + + private ImpactSet ResolveByPurlsCore(IEnumerable purls, bool usageOnly, Selector selector) + { + ArgumentNullException.ThrowIfNull(purls); + ArgumentNullException.ThrowIfNull(selector); + + var normalized = purls + .Where(static purl => !string.IsNullOrWhiteSpace(purl)) + .Select(static purl => purl.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (normalized.Length == 0) + { + return CreateEmptyImpactSet(selector, usageOnly); + } + + RoaringBitmap imageIds; + lock (_gate) + { + imageIds = RoaringBitmap.Create(Array.Empty()); + foreach (var purl in normalized) + { + if (_containsByPurl.TryGetValue(purl, out var bitmap)) + { + imageIds = imageIds | bitmap; + } + } + } + + return BuildImpactSet(imageIds, selector, usageOnly); + } + + private ImpactSet ResolveAllCore(Selector selector, bool usageOnly) + { + ArgumentNullException.ThrowIfNull(selector); + + RoaringBitmap bitmap; + lock (_gate) + { + var ids = _images.Keys.OrderBy(id => id).ToArray(); + bitmap = RoaringBitmap.Create(ids); + } + + return BuildImpactSet(bitmap, selector, usageOnly); + } + + private ImpactSet BuildImpactSet(RoaringBitmap imageIds, Selector selector, bool usageOnly) + { + var images = new List(); + var latestGeneratedAt = DateTimeOffset.MinValue; + + lock (_gate) + { + foreach (var imageId in imageIds) + { + if (!_images.TryGetValue(imageId, out var metadata)) + { + continue; + } + + if (!ImageMatchesSelector(metadata, selector)) + { + continue; + } + + if (usageOnly && metadata.EntrypointComponents.Length == 0) + { + continue; + } + + if (metadata.GeneratedAt > latestGeneratedAt) + { + latestGeneratedAt = metadata.GeneratedAt; + } + + images.Add(new ImpactImage( + metadata.Digest, + metadata.Registry, + metadata.Repository, + metadata.Namespaces, + metadata.Tags, + metadata.EntrypointComponents.Length > 0, + metadata.Labels)); + } + } + + if (images.Count == 0) + { + return CreateEmptyImpactSet(selector, usageOnly); + } + + images.Sort(static (left, right) => string.Compare(left.ImageDigest, right.ImageDigest, StringComparison.Ordinal)); + + var generatedAt = latestGeneratedAt == DateTimeOffset.MinValue ? _timeProvider.GetUtcNow() : latestGeneratedAt; + + return new ImpactSet( + selector, + images.ToImmutableArray(), + usageOnly, + generatedAt, + images.Count, + snapshotId: null, + schemaVersion: SchedulerSchemaVersions.ImpactSet); + } + + private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly) + { + return new ImpactSet( + selector, + ImmutableArray.Empty, + usageOnly, + _timeProvider.GetUtcNow(), + 0, + snapshotId: null, + schemaVersion: SchedulerSchemaVersions.ImpactSet); + } + + private static bool ImageMatchesSelector(ImpactImageRecord image, Selector selector) + { + if (selector.TenantId is not null && !string.Equals(selector.TenantId, image.TenantId, StringComparison.Ordinal)) + { + return false; + } + + if (!MatchesScope(image, selector)) + { + return false; + } + + if (selector.Digests.Length > 0 && !selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (selector.Repositories.Length > 0) + { + var repoMatch = selector.Repositories.Any(repo => + string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) || + string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)); + if (!repoMatch) + { + return false; + } + } + + if (selector.Namespaces.Length > 0) + { + if (image.Namespaces.IsDefaultOrEmpty) + { + return false; + } + + var namespaceMatch = selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)); + if (!namespaceMatch) + { + return false; + } + } + + if (selector.IncludeTags.Length > 0) + { + if (image.Tags.IsDefaultOrEmpty) + { + return false; + } + + var tagMatch = selector.IncludeTags.Any(pattern => image.Tags.Any(tag => MatchesTagPattern(tag, pattern))); + if (!tagMatch) + { + return false; + } + } + + if (selector.Labels.Length > 0) + { + if (image.Labels.Count == 0) + { + return false; + } + + foreach (var label in selector.Labels) + { + if (!image.Labels.TryGetValue(label.Key, out var value)) + { + return false; + } + + if (label.Values.Length > 0 && !label.Values.Contains(value, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + } + } + + return true; + } + + private void RemoveImageComponents(ImpactImageRecord record) + { + foreach (var key in record.Components) + { + if (_containsByPurl.TryGetValue(key, out var bitmap)) + { + var updated = RemoveImageFromBitmap(bitmap, record.ImageId); + if (updated is null) + { + _containsByPurl.Remove(key); + } + else + { + _containsByPurl[key] = updated; + } + } + } + + foreach (var key in record.EntrypointComponents) + { + if (_usedByEntrypointByPurl.TryGetValue(key, out var bitmap)) + { + var updated = RemoveImageFromBitmap(bitmap, record.ImageId); + if (updated is null) + { + _usedByEntrypointByPurl.Remove(key); + } + else + { + _usedByEntrypointByPurl[key] = updated; + } + } + } + } + + private static RoaringBitmap AddImageToBitmap(RoaringBitmap? bitmap, int imageId) + { + if (bitmap is null) + { + return RoaringBitmap.Create(new[] { imageId }); + } + + if (bitmap.Any(id => id == imageId)) + { + return bitmap; + } + + var merged = bitmap + .Concat(new[] { imageId }) + .Distinct() + .OrderBy(id => id) + .ToArray(); + + return RoaringBitmap.Create(merged); + } + + private static RoaringBitmap? RemoveImageFromBitmap(RoaringBitmap bitmap, int imageId) + { + var remaining = bitmap + .Where(id => id != imageId) + .OrderBy(id => id) + .ToArray(); + if (remaining.Length == 0) + { + return null; + } + + return RoaringBitmap.Create(remaining); + } + + private static bool MatchesScope(ImpactImageRecord image, Selector selector) + { + return selector.Scope switch + { + SelectorScope.AllImages => true, + SelectorScope.ByDigest => selector.Digests.Contains(image.Digest, StringComparer.OrdinalIgnoreCase), + SelectorScope.ByRepository => selector.Repositories.Any(repo => + string.Equals(repo, image.Repository, StringComparison.OrdinalIgnoreCase) || + string.Equals(repo, $"{image.Registry}/{image.Repository}", StringComparison.OrdinalIgnoreCase)), + SelectorScope.ByNamespace => !image.Namespaces.IsDefaultOrEmpty && selector.Namespaces.Any(ns => image.Namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)), + SelectorScope.ByLabels => selector.Labels.All(label => + image.Labels.TryGetValue(label.Key, out var value) && + (label.Values.Length == 0 || label.Values.Contains(value, StringComparer.OrdinalIgnoreCase))), + _ => true, + }; + } + + private static bool MatchesTagPattern(string tag, string pattern) + { + if (string.IsNullOrWhiteSpace(pattern)) + { + return false; + } + + if (pattern == "*") + { + return true; + } + + if (!pattern.Contains('*') && !pattern.Contains('?')) + { + return string.Equals(tag, pattern, StringComparison.OrdinalIgnoreCase); + } + + var escaped = Regex.Escape(pattern) + .Replace("\\*", ".*") + .Replace("\\?", "."); + return Regex.IsMatch(tag, $"^{escaped}$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + } + + private int EnsureImageId(string digest) + { + if (_imageIds.TryGetValue(digest, out var existing)) + { + return existing; + } + + var candidate = ComputeDeterministicId(digest); + while (_images.ContainsKey(candidate)) + { + candidate = (candidate + 1) & int.MaxValue; + if (candidate == 0) + { + candidate = 1; + } + } + + _imageIds[digest] = candidate; + return candidate; + } + + private static int ComputeDeterministicId(string digest) + { + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(digest)); + for (var offset = 0; offset <= bytes.Length - sizeof(int); offset += sizeof(int)) + { + var value = BinaryPrimitives.ReadInt32LittleEndian(bytes.AsSpan(offset, sizeof(int))) & int.MaxValue; + if (value != 0) + { + return value; + } + } + + return digest.GetHashCode(StringComparison.OrdinalIgnoreCase) & int.MaxValue; + } +} diff --git a/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj b/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj index 21dd08d0..c16068b1 100644 --- a/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj +++ b/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj @@ -4,8 +4,16 @@ enable enable + + + - + + + + + + diff --git a/src/StellaOps.Scheduler.ImpactIndex/TASKS.md b/src/StellaOps.Scheduler.ImpactIndex/TASKS.md index f248b877..951e9f24 100644 --- a/src/StellaOps.Scheduler.ImpactIndex/TASKS.md +++ b/src/StellaOps.Scheduler.ImpactIndex/TASKS.md @@ -3,8 +3,8 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCHED-IMPACT-16-300 | DONE (2025-10-20) | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. | -| SCHED-IMPACT-16-301 | TODO | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. | -| SCHED-IMPACT-16-302 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. | +| SCHED-IMPACT-16-301 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. | +| SCHED-IMPACT-16-302 | DONE (2025-10-26) | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. | | SCHED-IMPACT-16-303 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. | Snapshot routine implemented; invalidation tests pass; docs describe recovery. | > Removal tracking note: see `src/StellaOps.Scheduler.ImpactIndex/REMOVAL_NOTE.md` for follow-up actions once the roaring bitmap implementation lands. diff --git a/src/StellaOps.Scheduler.Models.Tests/GraphJobStateMachineTests.cs b/src/StellaOps.Scheduler.Models.Tests/GraphJobStateMachineTests.cs new file mode 100644 index 00000000..ae8c7391 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/GraphJobStateMachineTests.cs @@ -0,0 +1,171 @@ +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class GraphJobStateMachineTests +{ + [Theory] + [InlineData(GraphJobStatus.Pending, GraphJobStatus.Pending, true)] + [InlineData(GraphJobStatus.Pending, GraphJobStatus.Queued, true)] + [InlineData(GraphJobStatus.Pending, GraphJobStatus.Running, true)] + [InlineData(GraphJobStatus.Pending, GraphJobStatus.Completed, false)] + [InlineData(GraphJobStatus.Queued, GraphJobStatus.Running, true)] + [InlineData(GraphJobStatus.Queued, GraphJobStatus.Completed, false)] + [InlineData(GraphJobStatus.Running, GraphJobStatus.Completed, true)] + [InlineData(GraphJobStatus.Running, GraphJobStatus.Pending, false)] + [InlineData(GraphJobStatus.Completed, GraphJobStatus.Failed, false)] + public void CanTransition_ReturnsExpectedResult(GraphJobStatus from, GraphJobStatus to, bool expected) + { + Assert.Equal(expected, GraphJobStateMachine.CanTransition(from, to)); + } + + [Fact] + public void EnsureTransition_UpdatesBuildJobLifecycle() + { + var createdAt = new DateTimeOffset(2025, 10, 26, 12, 0, 0, TimeSpan.Zero); + var job = new GraphBuildJob( + id: "gbj_1", + tenantId: "tenant-alpha", + sbomId: "sbom_1", + sbomVersionId: "sbom_ver_1", + sbomDigest: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + status: GraphJobStatus.Pending, + trigger: GraphBuildJobTrigger.SbomVersion, + createdAt: createdAt); + + var queuedAt = createdAt.AddSeconds(5); + job = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Queued, queuedAt); + Assert.Equal(GraphJobStatus.Queued, job.Status); + Assert.Null(job.StartedAt); + Assert.Null(job.CompletedAt); + + var runningAt = queuedAt.AddSeconds(5); + job = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Running, runningAt, attempts: job.Attempts + 1); + Assert.Equal(GraphJobStatus.Running, job.Status); + Assert.Equal(runningAt, job.StartedAt); + Assert.Null(job.CompletedAt); + + var completedAt = runningAt.AddSeconds(30); + job = GraphJobStateMachine.EnsureTransition(job, GraphJobStatus.Completed, completedAt); + Assert.Equal(GraphJobStatus.Completed, job.Status); + Assert.Equal(runningAt, job.StartedAt); + Assert.Equal(completedAt, job.CompletedAt); + Assert.Null(job.Error); + } + + [Fact] + public void EnsureTransition_ToFailedRequiresError() + { + var job = new GraphBuildJob( + id: "gbj_1", + tenantId: "tenant-alpha", + sbomId: "sbom_1", + sbomVersionId: "sbom_ver_1", + sbomDigest: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + status: GraphJobStatus.Running, + trigger: GraphBuildJobTrigger.SbomVersion, + createdAt: DateTimeOffset.UtcNow, + startedAt: DateTimeOffset.UtcNow); + + Assert.Throws(() => GraphJobStateMachine.EnsureTransition( + job, + GraphJobStatus.Failed, + DateTimeOffset.UtcNow)); + } + + [Fact] + public void EnsureTransition_ToFailedSetsError() + { + var job = new GraphOverlayJob( + id: "goj_1", + tenantId: "tenant-alpha", + graphSnapshotId: "graph_snap_1", + overlayKind: GraphOverlayKind.Policy, + overlayKey: "policy@latest", + status: GraphJobStatus.Running, + trigger: GraphOverlayJobTrigger.Policy, + createdAt: DateTimeOffset.UtcNow, + startedAt: DateTimeOffset.UtcNow); + + var failed = GraphJobStateMachine.EnsureTransition( + job, + GraphJobStatus.Failed, + DateTimeOffset.UtcNow, + errorMessage: "cartographer timeout"); + + Assert.Equal(GraphJobStatus.Failed, failed.Status); + Assert.NotNull(failed.CompletedAt); + Assert.Equal("cartographer timeout", failed.Error); + } + + [Fact] + public void Validate_RequiresCompletedAtForTerminalState() + { + var job = new GraphOverlayJob( + id: "goj_1", + tenantId: "tenant-alpha", + graphSnapshotId: "graph_snap_1", + overlayKind: GraphOverlayKind.Policy, + overlayKey: "policy@latest", + status: GraphJobStatus.Completed, + trigger: GraphOverlayJobTrigger.Policy, + createdAt: DateTimeOffset.UtcNow); + + Assert.Throws(() => GraphJobStateMachine.Validate(job)); + } + + [Fact] + public void GraphOverlayJob_NormalizesSubjectsAndMetadata() + { + var createdAt = DateTimeOffset.UtcNow; + var job = new GraphOverlayJob( + id: "goj_norm", + tenantId: "tenant-alpha", + graphSnapshotId: "graph_snap_norm", + overlayKind: GraphOverlayKind.Policy, + overlayKey: "policy@norm", + status: GraphJobStatus.Pending, + trigger: GraphOverlayJobTrigger.Policy, + createdAt: createdAt, + subjects: new[] + { + "artifact/service-api", + "artifact/service-ui", + "artifact/service-api" + }, + metadata: new[] + { + new KeyValuePair("PolicyRunId", "run-123"), + new KeyValuePair("policyRunId", "run-123") + }); + + Assert.Equal(2, job.Subjects.Length); + Assert.Collection( + job.Subjects, + subject => Assert.Equal("artifact/service-api", subject), + subject => Assert.Equal("artifact/service-ui", subject)); + + Assert.Single(job.Metadata); + Assert.Equal("run-123", job.Metadata["policyrunid"]); + } + + [Fact] + public void GraphBuildJob_NormalizesDigestAndMetadata() + { + var job = new GraphBuildJob( + id: "gbj_norm", + tenantId: "tenant-alpha", + sbomId: "sbom_norm", + sbomVersionId: "sbom_ver_norm", + sbomDigest: "SHA256:ABCDEF1234567890ABCDEF1234567890ABCDEF1234567890ABCDEF1234567890", + status: GraphJobStatus.Pending, + trigger: GraphBuildJobTrigger.Manual, + createdAt: DateTimeOffset.UtcNow, + metadata: new[] + { + new KeyValuePair("SBoMEventId", "evt-42") + }); + + Assert.Equal("sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", job.SbomDigest); + Assert.Single(job.Metadata); + Assert.Equal("evt-42", job.Metadata["sbomeventid"]); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/PolicyRunModelsTests.cs b/src/StellaOps.Scheduler.Models.Tests/PolicyRunModelsTests.cs new file mode 100644 index 00000000..241b490d --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/PolicyRunModelsTests.cs @@ -0,0 +1,83 @@ +using System.Collections.Immutable; +using System.Text.Json; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class PolicyRunModelsTests +{ + [Fact] + public void PolicyRunInputs_NormalizesEnvironmentKeys() + { + var inputs = new PolicyRunInputs( + sbomSet: new[] { "sbom:two", "sbom:one" }, + env: new[] + { + new KeyValuePair("Sealed", true), + new KeyValuePair("Exposure", "internet"), + new KeyValuePair("region", JsonSerializer.SerializeToElement("global")) + }, + captureExplain: true); + + Assert.Equal(new[] { "sbom:one", "sbom:two" }, inputs.SbomSet); + Assert.True(inputs.CaptureExplain); + Assert.Equal(3, inputs.Environment.Count); + Assert.True(inputs.Environment.ContainsKey("sealed")); + Assert.Equal(JsonValueKind.True, inputs.Environment["sealed"].ValueKind); + Assert.Equal("internet", inputs.Environment["exposure"].GetString()); + Assert.Equal("global", inputs.Environment["region"].GetString()); + } + + [Fact] + public void PolicyRunStatus_ThrowsOnNegativeAttempts() + { + Assert.Throws(() => new PolicyRunStatus( + runId: "run:test", + tenantId: "tenant-alpha", + policyId: "P-1", + policyVersion: 1, + mode: PolicyRunMode.Full, + status: PolicyRunExecutionStatus.Queued, + priority: PolicyRunPriority.Normal, + queuedAt: DateTimeOffset.UtcNow, + attempts: -1)); + } + + [Fact] + public void PolicyDiffSummary_NormalizesSeverityKeys() + { + var summary = new PolicyDiffSummary( + added: 1, + removed: 2, + unchanged: 3, + bySeverity: new[] + { + new KeyValuePair("critical", new PolicyDiffSeverityDelta(1, 0)), + new KeyValuePair("HIGH", new PolicyDiffSeverityDelta(0, 1)) + }); + + Assert.True(summary.BySeverity.ContainsKey("Critical")); + Assert.True(summary.BySeverity.ContainsKey("High")); + } + + [Fact] + public void PolicyExplainTrace_LowercasesMetadataKeys() + { + var trace = new PolicyExplainTrace( + findingId: "finding:alpha", + policyId: "P-1", + policyVersion: 1, + tenantId: "tenant-alpha", + runId: "run:test", + verdict: new PolicyExplainVerdict(PolicyVerdictStatus.Passed, SeverityRank.Low, quiet: false, score: 0, rationale: "ok"), + evaluatedAt: DateTimeOffset.UtcNow, + metadata: ImmutableSortedDictionary.CreateRange(new[] + { + new KeyValuePair("TraceId", "trace-1"), + new KeyValuePair("ComponentPurl", "pkg:npm/a@1.0.0") + })); + + Assert.Equal("trace-1", trace.Metadata["traceid"]); + Assert.Equal("pkg:npm/a@1.0.0", trace.Metadata["componentpurl"]); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs b/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs index 284f83a0..a44e6bd4 100644 --- a/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs +++ b/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs @@ -33,11 +33,11 @@ public sealed class SamplePayloadTests } [Fact] - public void ImpactSetSample_RoundtripsThroughCanonicalSerializer() - { - var json = ReadSample("impact-set.json"); - var impact = CanonicalJsonSerializer.Deserialize(json); - + public void ImpactSetSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("impact-set.json"); + var impact = CanonicalJsonSerializer.Deserialize(json); + Assert.True(impact.UsageOnly); Assert.Single(impact.Images); @@ -54,9 +54,95 @@ public sealed class SamplePayloadTests Assert.Equal("scheduler", audit.Category); Assert.Equal("pause", audit.Action); - var canonical = CanonicalJsonSerializer.Serialize(audit); - AssertJsonEquivalent(json, canonical); - } + var canonical = CanonicalJsonSerializer.Serialize(audit); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void GraphBuildJobSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("graph-build-job.json"); + var job = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal(GraphJobStatus.Running, job.Status); + Assert.Equal(GraphBuildJobTrigger.SbomVersion, job.Trigger); + + var canonical = CanonicalJsonSerializer.Serialize(job); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void GraphOverlayJobSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("graph-overlay-job.json"); + var job = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal(GraphJobStatus.Queued, job.Status); + Assert.Equal(GraphOverlayJobTrigger.Policy, job.Trigger); + Assert.Equal(GraphOverlayKind.Policy, job.OverlayKind); + + var canonical = CanonicalJsonSerializer.Serialize(job); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void PolicyRunRequestSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("policy-run-request.json"); + var request = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal("default", request.TenantId); + Assert.Equal(PolicyRunMode.Incremental, request.Mode); + Assert.Equal("run:P-7:2025-10-26:auto", request.RunId); + Assert.True(request.Inputs.CaptureExplain); + Assert.Equal(2, request.Inputs.SbomSet.Length); + + var canonical = CanonicalJsonSerializer.Serialize(request); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void PolicyRunStatusSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("policy-run-status.json"); + var status = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal(PolicyRunExecutionStatus.Succeeded, status.Status); + Assert.Equal(1742, status.Stats.Components); + Assert.Equal("scheduler", status.Metadata["orchestrator"]); + + var canonical = CanonicalJsonSerializer.Serialize(status); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void PolicyDiffSummarySample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("policy-diff-summary.json"); + var summary = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal(12, summary.Added); + Assert.Contains(summary.BySeverity.Keys, key => string.Equals(key, "critical", StringComparison.OrdinalIgnoreCase)); + Assert.Equal(2, summary.RuleHits.Length); + + var canonical = CanonicalJsonSerializer.Serialize(summary); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void PolicyExplainTraceSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("policy-explain-trace.json"); + var trace = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal("finding:sbom:S-42/pkg:npm/lodash@4.17.21", trace.FindingId); + Assert.Equal(PolicyVerdictStatus.Blocked, trace.Verdict.Status); + Assert.Equal(2, trace.RuleChain.Length); + Assert.Equal(2, trace.Evidence.Length); + + var canonical = CanonicalJsonSerializer.Serialize(trace); + AssertJsonEquivalent(json, canonical); + } private static string ReadSample(string fileName) { diff --git a/src/StellaOps.Scheduler.Models/AssemblyInfo.cs b/src/StellaOps.Scheduler.Models/AssemblyInfo.cs new file mode 100644 index 00000000..1752efa5 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scheduler.ImpactIndex")] diff --git a/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs b/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs index 722a2ef2..762b5727 100644 --- a/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs +++ b/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs @@ -97,11 +97,11 @@ public static class CanonicalJsonSerializer "newHigh", "newMedium", "newLow", - }, - [typeof(RunReason)] = new[] - { - "manualReason", - "feedserExportId", + }, + [typeof(RunReason)] = new[] + { + "manualReason", + "feedserExportId", "vexerExportId", "cursor", "impactWindowFrom", @@ -136,11 +136,11 @@ public static class CanonicalJsonSerializer "usageOnly", "generatedAt", "total", - "snapshotId", - }, - [typeof(ImpactImage)] = new[] - { - "imageDigest", + "snapshotId", + }, + [typeof(ImpactImage)] = new[] + { + "imageDigest", "registry", "repository", "namespaces", @@ -163,13 +163,189 @@ public static class CanonicalJsonSerializer "metadata", "message", }, - [typeof(AuditActor)] = new[] - { - "actorId", - "displayName", - "kind", - }, - }; + [typeof(AuditActor)] = new[] + { + "actorId", + "displayName", + "kind", + }, + [typeof(GraphBuildJob)] = new[] + { + "schemaVersion", + "id", + "tenantId", + "sbomId", + "sbomVersionId", + "sbomDigest", + "graphSnapshotId", + "status", + "trigger", + "attempts", + "cartographerJobId", + "correlationId", + "createdAt", + "startedAt", + "completedAt", + "error", + "metadata", + }, + [typeof(GraphOverlayJob)] = new[] + { + "schemaVersion", + "id", + "tenantId", + "graphSnapshotId", + "buildJobId", + "overlayKind", + "overlayKey", + "subjects", + "status", + "trigger", + "attempts", + "correlationId", + "createdAt", + "startedAt", + "completedAt", + "error", + "metadata", + }, + [typeof(PolicyRunRequest)] = new[] + { + "schemaVersion", + "tenantId", + "policyId", + "policyVersion", + "mode", + "priority", + "runId", + "queuedAt", + "requestedBy", + "correlationId", + "metadata", + "inputs", + }, + [typeof(PolicyRunInputs)] = new[] + { + "sbomSet", + "advisoryCursor", + "vexCursor", + "environment", + "captureExplain", + }, + [typeof(PolicyRunStatus)] = new[] + { + "schemaVersion", + "runId", + "tenantId", + "policyId", + "policyVersion", + "mode", + "status", + "priority", + "queuedAt", + "startedAt", + "finishedAt", + "determinismHash", + "errorCode", + "error", + "attempts", + "traceId", + "explainUri", + "metadata", + "stats", + "inputs", + }, + [typeof(PolicyRunStats)] = new[] + { + "components", + "rulesFired", + "findingsWritten", + "vexOverrides", + "quieted", + "suppressed", + "durationSeconds", + }, + [typeof(PolicyDiffSummary)] = new[] + { + "schemaVersion", + "added", + "removed", + "unchanged", + "bySeverity", + "ruleHits", + }, + [typeof(PolicyDiffSeverityDelta)] = new[] + { + "up", + "down", + }, + [typeof(PolicyDiffRuleDelta)] = new[] + { + "ruleId", + "ruleName", + "up", + "down", + }, + [typeof(PolicyExplainTrace)] = new[] + { + "schemaVersion", + "findingId", + "policyId", + "policyVersion", + "tenantId", + "runId", + "evaluatedAt", + "verdict", + "ruleChain", + "evidence", + "vexImpacts", + "history", + "metadata", + }, + [typeof(PolicyExplainVerdict)] = new[] + { + "status", + "severity", + "quiet", + "score", + "rationale", + }, + [typeof(PolicyExplainRule)] = new[] + { + "ruleId", + "ruleName", + "action", + "decision", + "score", + "condition", + }, + [typeof(PolicyExplainEvidence)] = new[] + { + "type", + "reference", + "source", + "status", + "weight", + "justification", + "metadata", + }, + [typeof(PolicyExplainVexImpact)] = new[] + { + "statementId", + "provider", + "status", + "accepted", + "justification", + "confidence", + }, + [typeof(PolicyExplainHistoryEvent)] = new[] + { + "status", + "occurredAt", + "actor", + "note", + }, + }; public static string Serialize(T value) => JsonSerializer.Serialize(value, CompactOptions); @@ -196,11 +372,19 @@ public static class CanonicalJsonSerializer options.TypeInfoResolver = new DeterministicResolver(resolver); options.Converters.Add(new ScheduleModeConverter()); options.Converters.Add(new SelectorScopeConverter()); - options.Converters.Add(new RunTriggerConverter()); - options.Converters.Add(new RunStateConverter()); - options.Converters.Add(new SeverityRankConverter()); - return options; - } + options.Converters.Add(new RunTriggerConverter()); + options.Converters.Add(new RunStateConverter()); + options.Converters.Add(new SeverityRankConverter()); + options.Converters.Add(new GraphJobStatusConverter()); + options.Converters.Add(new GraphBuildJobTriggerConverter()); + options.Converters.Add(new GraphOverlayJobTriggerConverter()); + options.Converters.Add(new GraphOverlayKindConverter()); + options.Converters.Add(new PolicyRunModeConverter()); + options.Converters.Add(new PolicyRunPriorityConverter()); + options.Converters.Add(new PolicyRunExecutionStatusConverter()); + options.Converters.Add(new PolicyVerdictStatusConverter()); + return options; + } private sealed class DeterministicResolver : IJsonTypeInfoResolver { diff --git a/src/StellaOps.Scheduler.Models/EnumConverters.cs b/src/StellaOps.Scheduler.Models/EnumConverters.cs index e3d1d6d0..ab723520 100644 --- a/src/StellaOps.Scheduler.Models/EnumConverters.cs +++ b/src/StellaOps.Scheduler.Models/EnumConverters.cs @@ -32,26 +32,114 @@ internal sealed class RunStateConverter : LowerCaseEnumConverter { } -internal sealed class SeverityRankConverter : LowerCaseEnumConverter -{ - protected override string ConvertToString(SeverityRank value) - => value switch - { +internal sealed class SeverityRankConverter : LowerCaseEnumConverter +{ + protected override string ConvertToString(SeverityRank value) + => value switch + { SeverityRank.None => "none", SeverityRank.Info => "info", SeverityRank.Low => "low", SeverityRank.Medium => "medium", SeverityRank.High => "high", SeverityRank.Critical => "critical", - SeverityRank.Unknown => "unknown", - _ => throw new ArgumentOutOfRangeException(nameof(value), value, null), - }; -} - -internal abstract class HyphenatedEnumConverter : JsonConverter - where TEnum : struct, Enum -{ - private readonly Dictionary _reverse; + SeverityRank.Unknown => "unknown", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, null), + }; +} + +internal sealed class GraphJobStatusConverter : LowerCaseEnumConverter +{ +} + +internal sealed class GraphBuildJobTriggerConverter : HyphenatedEnumConverter +{ + protected override IReadOnlyDictionary Map { get; } = new Dictionary + { + [GraphBuildJobTrigger.SbomVersion] = "sbom-version", + [GraphBuildJobTrigger.Backfill] = "backfill", + [GraphBuildJobTrigger.Manual] = "manual", + }; +} + +internal sealed class GraphOverlayJobTriggerConverter : HyphenatedEnumConverter +{ + protected override IReadOnlyDictionary Map { get; } = new Dictionary + { + [GraphOverlayJobTrigger.Policy] = "policy", + [GraphOverlayJobTrigger.Advisory] = "advisory", + [GraphOverlayJobTrigger.Vex] = "vex", + [GraphOverlayJobTrigger.SbomVersion] = "sbom-version", + [GraphOverlayJobTrigger.Manual] = "manual", + }; +} + +internal sealed class GraphOverlayKindConverter : LowerCaseEnumConverter +{ +} + +internal sealed class PolicyRunModeConverter : LowerCaseEnumConverter +{ +} + +internal sealed class PolicyRunPriorityConverter : LowerCaseEnumConverter +{ +} + +internal sealed class PolicyRunExecutionStatusConverter : JsonConverter +{ + private static readonly IReadOnlyDictionary Reverse = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["queued"] = PolicyRunExecutionStatus.Queued, + ["running"] = PolicyRunExecutionStatus.Running, + ["succeeded"] = PolicyRunExecutionStatus.Succeeded, + ["failed"] = PolicyRunExecutionStatus.Failed, + ["canceled"] = PolicyRunExecutionStatus.Cancelled, + ["cancelled"] = PolicyRunExecutionStatus.Cancelled, + ["replay_pending"] = PolicyRunExecutionStatus.ReplayPending, + ["replay-pending"] = PolicyRunExecutionStatus.ReplayPending, + }; + + private static readonly IReadOnlyDictionary Forward = new Dictionary + { + [PolicyRunExecutionStatus.Queued] = "queued", + [PolicyRunExecutionStatus.Running] = "running", + [PolicyRunExecutionStatus.Succeeded] = "succeeded", + [PolicyRunExecutionStatus.Failed] = "failed", + [PolicyRunExecutionStatus.Cancelled] = "canceled", + [PolicyRunExecutionStatus.ReplayPending] = "replay_pending", + }; + + public override PolicyRunExecutionStatus Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = reader.GetString(); + if (value is not null && Reverse.TryGetValue(value, out var status)) + { + return status; + } + + throw new JsonException($"Value '{value}' is not a valid {nameof(PolicyRunExecutionStatus)}."); + } + + public override void Write(Utf8JsonWriter writer, PolicyRunExecutionStatus value, JsonSerializerOptions options) + { + if (!Forward.TryGetValue(value, out var text)) + { + throw new JsonException($"Unable to serialize {nameof(PolicyRunExecutionStatus)} value '{value}'."); + } + + writer.WriteStringValue(text); + } +} + +internal sealed class PolicyVerdictStatusConverter : LowerCaseEnumConverter +{ +} + +internal abstract class HyphenatedEnumConverter : JsonConverter + where TEnum : struct, Enum +{ + private readonly Dictionary _reverse; protected HyphenatedEnumConverter() { diff --git a/src/StellaOps.Scheduler.Models/Enums.cs b/src/StellaOps.Scheduler.Models/Enums.cs index dfdabe8c..e9071e2d 100644 --- a/src/StellaOps.Scheduler.Models/Enums.cs +++ b/src/StellaOps.Scheduler.Models/Enums.cs @@ -54,14 +54,112 @@ public enum RunState /// /// Severity rankings used in scheduler payloads. /// -[JsonConverter(typeof(SeverityRankConverter))] -public enum SeverityRank -{ - None = 0, - Info = 1, +[JsonConverter(typeof(SeverityRankConverter))] +public enum SeverityRank +{ + None = 0, + Info = 1, Low = 2, Medium = 3, - High = 4, - Critical = 5, - Unknown = 6, -} + High = 4, + Critical = 5, + Unknown = 6, +} + +/// +/// Status lifecycle shared by graph build and overlay jobs. +/// +[JsonConverter(typeof(GraphJobStatusConverter))] +public enum GraphJobStatus +{ + Pending, + Queued, + Running, + Completed, + Failed, + Cancelled, +} + +/// +/// Trigger indicating why a graph build job was enqueued. +/// +[JsonConverter(typeof(GraphBuildJobTriggerConverter))] +public enum GraphBuildJobTrigger +{ + SbomVersion, + Backfill, + Manual, +} + +/// +/// Trigger indicating why a graph overlay job was enqueued. +/// +[JsonConverter(typeof(GraphOverlayJobTriggerConverter))] +public enum GraphOverlayJobTrigger +{ + Policy, + Advisory, + Vex, + SbomVersion, + Manual, +} + +/// +/// Overlay category applied to a graph snapshot. +/// +[JsonConverter(typeof(GraphOverlayKindConverter))] +public enum GraphOverlayKind +{ + Policy, + Advisory, + Vex, +} + +/// +/// Mode for policy runs executed by the Policy Engine. +/// +[JsonConverter(typeof(PolicyRunModeConverter))] +public enum PolicyRunMode +{ + Full, + Incremental, + Simulate, +} + +/// +/// Priority assigned to a policy run request. +/// +[JsonConverter(typeof(PolicyRunPriorityConverter))] +public enum PolicyRunPriority +{ + Normal, + High, + Emergency, +} + +/// +/// Execution status for policy runs tracked in policy_runs. +/// +[JsonConverter(typeof(PolicyRunExecutionStatusConverter))] +public enum PolicyRunExecutionStatus +{ + Queued, + Running, + Succeeded, + Failed, + Cancelled, + ReplayPending, +} + +/// +/// Resulting verdict for a policy evaluation. +/// +[JsonConverter(typeof(PolicyVerdictStatusConverter))] +public enum PolicyVerdictStatus +{ + Passed, + Warned, + Blocked, + Quieted, + Ignored, +} diff --git a/src/StellaOps.Scheduler.Models/GraphBuildJob.cs b/src/StellaOps.Scheduler.Models/GraphBuildJob.cs new file mode 100644 index 00000000..82ed6b62 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/GraphBuildJob.cs @@ -0,0 +1,132 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Job instructing Cartographer to materialize a graph snapshot for an SBOM version. +/// +public sealed record GraphBuildJob +{ + public GraphBuildJob( + string id, + string tenantId, + string sbomId, + string sbomVersionId, + string sbomDigest, + GraphJobStatus status, + GraphBuildJobTrigger trigger, + DateTimeOffset createdAt, + string? graphSnapshotId = null, + int attempts = 0, + string? cartographerJobId = null, + string? correlationId = null, + DateTimeOffset? startedAt = null, + DateTimeOffset? completedAt = null, + string? error = null, + IEnumerable>? metadata = null, + string? schemaVersion = null) + : this( + id, + tenantId, + sbomId, + sbomVersionId, + sbomDigest, + Validation.TrimToNull(graphSnapshotId), + status, + trigger, + Validation.EnsureNonNegative(attempts, nameof(attempts)), + Validation.TrimToNull(cartographerJobId), + Validation.TrimToNull(correlationId), + Validation.NormalizeTimestamp(createdAt), + Validation.NormalizeTimestamp(startedAt), + Validation.NormalizeTimestamp(completedAt), + Validation.TrimToNull(error), + Validation.NormalizeMetadata(metadata), + schemaVersion) + { + } + + [JsonConstructor] + public GraphBuildJob( + string id, + string tenantId, + string sbomId, + string sbomVersionId, + string sbomDigest, + string? graphSnapshotId, + GraphJobStatus status, + GraphBuildJobTrigger trigger, + int attempts, + string? cartographerJobId, + string? correlationId, + DateTimeOffset createdAt, + DateTimeOffset? startedAt, + DateTimeOffset? completedAt, + string? error, + ImmutableSortedDictionary metadata, + string? schemaVersion = null) + { + Id = Validation.EnsureId(id, nameof(id)); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + SbomId = Validation.EnsureId(sbomId, nameof(sbomId)); + SbomVersionId = Validation.EnsureId(sbomVersionId, nameof(sbomVersionId)); + SbomDigest = Validation.EnsureDigestFormat(sbomDigest, nameof(sbomDigest)); + GraphSnapshotId = Validation.TrimToNull(graphSnapshotId); + Status = status; + Trigger = trigger; + Attempts = Validation.EnsureNonNegative(attempts, nameof(attempts)); + CartographerJobId = Validation.TrimToNull(cartographerJobId); + CorrelationId = Validation.TrimToNull(correlationId); + CreatedAt = Validation.NormalizeTimestamp(createdAt); + StartedAt = Validation.NormalizeTimestamp(startedAt); + CompletedAt = Validation.NormalizeTimestamp(completedAt); + Error = Validation.TrimToNull(error); + var materializedMetadata = metadata ?? ImmutableSortedDictionary.Empty; + Metadata = materializedMetadata.Count > 0 + ? materializedMetadata.WithComparers(StringComparer.Ordinal) + : ImmutableSortedDictionary.Empty; + SchemaVersion = SchedulerSchemaVersions.EnsureGraphBuildJob(schemaVersion); + } + + public string SchemaVersion { get; } + + public string Id { get; } + + public string TenantId { get; } + + public string SbomId { get; } + + public string SbomVersionId { get; } + + public string SbomDigest { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? GraphSnapshotId { get; init; } + + public GraphJobStatus Status { get; init; } + + public GraphBuildJobTrigger Trigger { get; } + + public int Attempts { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CartographerJobId { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CorrelationId { get; init; } + + public DateTimeOffset CreatedAt { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? StartedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? CompletedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Metadata { get; } = ImmutableSortedDictionary.Empty; +} diff --git a/src/StellaOps.Scheduler.Models/GraphJobStateMachine.cs b/src/StellaOps.Scheduler.Models/GraphJobStateMachine.cs new file mode 100644 index 00000000..c529b9b8 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/GraphJobStateMachine.cs @@ -0,0 +1,241 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Scheduler.Models; + +/// +/// Encapsulates allowed status transitions and invariants for graph jobs. +/// +public static class GraphJobStateMachine +{ + private static readonly IReadOnlyDictionary Adjacency = new Dictionary + { + [GraphJobStatus.Pending] = new[] { GraphJobStatus.Pending, GraphJobStatus.Queued, GraphJobStatus.Running, GraphJobStatus.Failed, GraphJobStatus.Cancelled }, + [GraphJobStatus.Queued] = new[] { GraphJobStatus.Queued, GraphJobStatus.Running, GraphJobStatus.Failed, GraphJobStatus.Cancelled }, + [GraphJobStatus.Running] = new[] { GraphJobStatus.Running, GraphJobStatus.Completed, GraphJobStatus.Failed, GraphJobStatus.Cancelled }, + [GraphJobStatus.Completed] = new[] { GraphJobStatus.Completed }, + [GraphJobStatus.Failed] = new[] { GraphJobStatus.Failed }, + [GraphJobStatus.Cancelled] = new[] { GraphJobStatus.Cancelled }, + }; + + public static bool CanTransition(GraphJobStatus from, GraphJobStatus to) + { + if (!Adjacency.TryGetValue(from, out var allowed)) + { + return false; + } + + return allowed.Contains(to); + } + + public static bool IsTerminal(GraphJobStatus status) + => status is GraphJobStatus.Completed or GraphJobStatus.Failed or GraphJobStatus.Cancelled; + + public static GraphBuildJob EnsureTransition( + GraphBuildJob job, + GraphJobStatus next, + DateTimeOffset timestamp, + int? attempts = null, + string? errorMessage = null) + { + ArgumentNullException.ThrowIfNull(job); + + var normalizedTimestamp = Validation.NormalizeTimestamp(timestamp); + var current = job.Status; + + if (!CanTransition(current, next)) + { + throw new InvalidOperationException($"Graph build job transition from '{current}' to '{next}' is not allowed."); + } + + var nextAttempts = attempts ?? job.Attempts; + if (nextAttempts < job.Attempts) + { + throw new InvalidOperationException("Graph job attempts cannot decrease."); + } + + var startedAt = job.StartedAt; + var completedAt = job.CompletedAt; + + if (current != GraphJobStatus.Running && next == GraphJobStatus.Running && startedAt is null) + { + startedAt = normalizedTimestamp; + } + + if (IsTerminal(next)) + { + completedAt ??= normalizedTimestamp; + } + + string? nextError = null; + if (next == GraphJobStatus.Failed) + { + var effectiveError = string.IsNullOrWhiteSpace(errorMessage) ? job.Error : errorMessage.Trim(); + if (string.IsNullOrWhiteSpace(effectiveError)) + { + throw new InvalidOperationException("Transitioning to Failed requires a non-empty error message."); + } + + nextError = effectiveError; + } + else if (!string.IsNullOrWhiteSpace(errorMessage)) + { + throw new InvalidOperationException("Error message can only be provided when transitioning to Failed state."); + } + + var updated = job with + { + Status = next, + Attempts = nextAttempts, + StartedAt = startedAt, + CompletedAt = completedAt, + Error = nextError, + }; + + Validate(updated); + return updated; + } + + public static GraphOverlayJob EnsureTransition( + GraphOverlayJob job, + GraphJobStatus next, + DateTimeOffset timestamp, + int? attempts = null, + string? errorMessage = null) + { + ArgumentNullException.ThrowIfNull(job); + + var normalizedTimestamp = Validation.NormalizeTimestamp(timestamp); + var current = job.Status; + + if (!CanTransition(current, next)) + { + throw new InvalidOperationException($"Graph overlay job transition from '{current}' to '{next}' is not allowed."); + } + + var nextAttempts = attempts ?? job.Attempts; + if (nextAttempts < job.Attempts) + { + throw new InvalidOperationException("Graph job attempts cannot decrease."); + } + + var startedAt = job.StartedAt; + var completedAt = job.CompletedAt; + + if (current != GraphJobStatus.Running && next == GraphJobStatus.Running && startedAt is null) + { + startedAt = normalizedTimestamp; + } + + if (IsTerminal(next)) + { + completedAt ??= normalizedTimestamp; + } + + string? nextError = null; + if (next == GraphJobStatus.Failed) + { + var effectiveError = string.IsNullOrWhiteSpace(errorMessage) ? job.Error : errorMessage.Trim(); + if (string.IsNullOrWhiteSpace(effectiveError)) + { + throw new InvalidOperationException("Transitioning to Failed requires a non-empty error message."); + } + + nextError = effectiveError; + } + else if (!string.IsNullOrWhiteSpace(errorMessage)) + { + throw new InvalidOperationException("Error message can only be provided when transitioning to Failed state."); + } + + var updated = job with + { + Status = next, + Attempts = nextAttempts, + StartedAt = startedAt, + CompletedAt = completedAt, + Error = nextError, + }; + + Validate(updated); + return updated; + } + + public static void Validate(GraphBuildJob job) + { + ArgumentNullException.ThrowIfNull(job); + + if (job.StartedAt is { } started && started < job.CreatedAt) + { + throw new InvalidOperationException("GraphBuildJob.StartedAt cannot be earlier than CreatedAt."); + } + + if (job.CompletedAt is { } completed) + { + if (job.StartedAt is { } start && completed < start) + { + throw new InvalidOperationException("GraphBuildJob.CompletedAt cannot be earlier than StartedAt."); + } + + if (!IsTerminal(job.Status)) + { + throw new InvalidOperationException("GraphBuildJob.CompletedAt set while status is not terminal."); + } + } + else if (IsTerminal(job.Status)) + { + throw new InvalidOperationException("Terminal graph build job states must include CompletedAt."); + } + + if (job.Status == GraphJobStatus.Failed) + { + if (string.IsNullOrWhiteSpace(job.Error)) + { + throw new InvalidOperationException("GraphBuildJob.Error must be populated when status is Failed."); + } + } + else if (!string.IsNullOrWhiteSpace(job.Error)) + { + throw new InvalidOperationException("GraphBuildJob.Error must be null for non-failed states."); + } + } + + public static void Validate(GraphOverlayJob job) + { + ArgumentNullException.ThrowIfNull(job); + + if (job.StartedAt is { } started && started < job.CreatedAt) + { + throw new InvalidOperationException("GraphOverlayJob.StartedAt cannot be earlier than CreatedAt."); + } + + if (job.CompletedAt is { } completed) + { + if (job.StartedAt is { } start && completed < start) + { + throw new InvalidOperationException("GraphOverlayJob.CompletedAt cannot be earlier than StartedAt."); + } + + if (!IsTerminal(job.Status)) + { + throw new InvalidOperationException("GraphOverlayJob.CompletedAt set while status is not terminal."); + } + } + else if (IsTerminal(job.Status)) + { + throw new InvalidOperationException("Terminal graph overlay job states must include CompletedAt."); + } + + if (job.Status == GraphJobStatus.Failed) + { + if (string.IsNullOrWhiteSpace(job.Error)) + { + throw new InvalidOperationException("GraphOverlayJob.Error must be populated when status is Failed."); + } + } + else if (!string.IsNullOrWhiteSpace(job.Error)) + { + throw new InvalidOperationException("GraphOverlayJob.Error must be null for non-failed states."); + } + } +} diff --git a/src/StellaOps.Scheduler.Models/GraphOverlayJob.cs b/src/StellaOps.Scheduler.Models/GraphOverlayJob.cs new file mode 100644 index 00000000..b7e67b98 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/GraphOverlayJob.cs @@ -0,0 +1,132 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Job that materializes or refreshes an overlay on top of an existing graph snapshot. +/// +public sealed record GraphOverlayJob +{ + public GraphOverlayJob( + string id, + string tenantId, + string graphSnapshotId, + GraphOverlayKind overlayKind, + string overlayKey, + GraphJobStatus status, + GraphOverlayJobTrigger trigger, + DateTimeOffset createdAt, + IEnumerable? subjects = null, + int attempts = 0, + string? buildJobId = null, + string? correlationId = null, + DateTimeOffset? startedAt = null, + DateTimeOffset? completedAt = null, + string? error = null, + IEnumerable>? metadata = null, + string? schemaVersion = null) + : this( + id, + tenantId, + graphSnapshotId, + Validation.TrimToNull(buildJobId), + overlayKind, + Validation.EnsureNotNullOrWhiteSpace(overlayKey, nameof(overlayKey)), + Validation.NormalizeStringSet(subjects, nameof(subjects)), + status, + trigger, + Validation.EnsureNonNegative(attempts, nameof(attempts)), + Validation.TrimToNull(correlationId), + Validation.NormalizeTimestamp(createdAt), + Validation.NormalizeTimestamp(startedAt), + Validation.NormalizeTimestamp(completedAt), + Validation.TrimToNull(error), + Validation.NormalizeMetadata(metadata), + schemaVersion) + { + } + + [JsonConstructor] + public GraphOverlayJob( + string id, + string tenantId, + string graphSnapshotId, + string? buildJobId, + GraphOverlayKind overlayKind, + string overlayKey, + ImmutableArray subjects, + GraphJobStatus status, + GraphOverlayJobTrigger trigger, + int attempts, + string? correlationId, + DateTimeOffset createdAt, + DateTimeOffset? startedAt, + DateTimeOffset? completedAt, + string? error, + ImmutableSortedDictionary metadata, + string? schemaVersion = null) + { + Id = Validation.EnsureId(id, nameof(id)); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + GraphSnapshotId = Validation.EnsureId(graphSnapshotId, nameof(graphSnapshotId)); + BuildJobId = Validation.TrimToNull(buildJobId); + OverlayKind = overlayKind; + OverlayKey = Validation.EnsureNotNullOrWhiteSpace(overlayKey, nameof(overlayKey)); + Subjects = subjects.IsDefault ? ImmutableArray.Empty : subjects; + Status = status; + Trigger = trigger; + Attempts = Validation.EnsureNonNegative(attempts, nameof(attempts)); + CorrelationId = Validation.TrimToNull(correlationId); + CreatedAt = Validation.NormalizeTimestamp(createdAt); + StartedAt = Validation.NormalizeTimestamp(startedAt); + CompletedAt = Validation.NormalizeTimestamp(completedAt); + Error = Validation.TrimToNull(error); + var materializedMetadata = metadata ?? ImmutableSortedDictionary.Empty; + Metadata = materializedMetadata.Count > 0 + ? materializedMetadata.WithComparers(StringComparer.Ordinal) + : ImmutableSortedDictionary.Empty; + SchemaVersion = SchedulerSchemaVersions.EnsureGraphOverlayJob(schemaVersion); + } + + public string SchemaVersion { get; } + + public string Id { get; } + + public string TenantId { get; } + + public string GraphSnapshotId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? BuildJobId { get; init; } + + public GraphOverlayKind OverlayKind { get; } + + public string OverlayKey { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Subjects { get; } = ImmutableArray.Empty; + + public GraphJobStatus Status { get; init; } + + public GraphOverlayJobTrigger Trigger { get; } + + public int Attempts { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CorrelationId { get; init; } + + public DateTimeOffset CreatedAt { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? StartedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? CompletedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Metadata { get; } = ImmutableSortedDictionary.Empty; +} diff --git a/src/StellaOps.Scheduler.Models/PolicyRunModels.cs b/src/StellaOps.Scheduler.Models/PolicyRunModels.cs new file mode 100644 index 00000000..2f5d8249 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/PolicyRunModels.cs @@ -0,0 +1,930 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Request payload enqueued by the policy orchestrator/clients. +/// +public sealed record PolicyRunRequest +{ + public PolicyRunRequest( + string tenantId, + string policyId, + PolicyRunMode mode, + PolicyRunInputs? inputs = null, + PolicyRunPriority priority = PolicyRunPriority.Normal, + string? runId = null, + int? policyVersion = null, + string? requestedBy = null, + DateTimeOffset? queuedAt = null, + string? correlationId = null, + ImmutableSortedDictionary? metadata = null, + string? schemaVersion = null) + : this( + tenantId, + policyId, + policyVersion, + mode, + priority, + runId, + Validation.NormalizeTimestamp(queuedAt), + Validation.TrimToNull(requestedBy), + Validation.TrimToNull(correlationId), + metadata ?? ImmutableSortedDictionary.Empty, + inputs ?? PolicyRunInputs.Empty, + schemaVersion) + { + } + + [JsonConstructor] + public PolicyRunRequest( + string tenantId, + string policyId, + int? policyVersion, + PolicyRunMode mode, + PolicyRunPriority priority, + string? runId, + DateTimeOffset? queuedAt, + string? requestedBy, + string? correlationId, + ImmutableSortedDictionary metadata, + PolicyRunInputs inputs, + string? schemaVersion = null) + { + SchemaVersion = SchedulerSchemaVersions.EnsurePolicyRunRequest(schemaVersion); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + PolicyId = Validation.EnsureSimpleIdentifier(policyId, nameof(policyId)); + if (policyVersion is not null && policyVersion <= 0) + { + throw new ArgumentOutOfRangeException(nameof(policyVersion), policyVersion, "Policy version must be positive."); + } + + PolicyVersion = policyVersion; + Mode = mode; + Priority = priority; + RunId = Validation.TrimToNull(runId) is { Length: > 0 } normalizedRunId + ? Validation.EnsureId(normalizedRunId, nameof(runId)) + : null; + QueuedAt = Validation.NormalizeTimestamp(queuedAt); + RequestedBy = Validation.TrimToNull(requestedBy); + CorrelationId = Validation.TrimToNull(correlationId); + var normalizedMetadata = (metadata ?? ImmutableSortedDictionary.Empty) + .Select(static pair => new KeyValuePair( + Validation.TrimToNull(pair.Key)?.ToLowerInvariant() ?? string.Empty, + Validation.TrimToNull(pair.Value) ?? string.Empty)) + .Where(static pair => !string.IsNullOrEmpty(pair.Key) && !string.IsNullOrEmpty(pair.Value)) + .DistinctBy(static pair => pair.Key, StringComparer.Ordinal) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); + Metadata = normalizedMetadata.Count == 0 ? null : normalizedMetadata; + Inputs = inputs ?? PolicyRunInputs.Empty; + } + + public string SchemaVersion { get; } + + public string TenantId { get; } + + public string PolicyId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? PolicyVersion { get; } + + public PolicyRunMode Mode { get; } + + public PolicyRunPriority Priority { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? RunId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? QueuedAt { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? RequestedBy { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CorrelationId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ImmutableSortedDictionary? Metadata { get; } + + public PolicyRunInputs Inputs { get; } = PolicyRunInputs.Empty; +} + +/// +/// Scoped inputs for policy runs (SBOM set, cursors, environment). +/// +public sealed record PolicyRunInputs +{ + public static PolicyRunInputs Empty { get; } = new(); + + public PolicyRunInputs( + IEnumerable? sbomSet = null, + DateTimeOffset? advisoryCursor = null, + DateTimeOffset? vexCursor = null, + IEnumerable>? env = null, + bool captureExplain = false) + { + _sbomSet = NormalizeSbomSet(sbomSet); + _advisoryCursor = Validation.NormalizeTimestamp(advisoryCursor); + _vexCursor = Validation.NormalizeTimestamp(vexCursor); + _environment = NormalizeEnvironment(env); + CaptureExplain = captureExplain; + } + + public PolicyRunInputs() + { + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray SbomSet + { + get => _sbomSet; + init => _sbomSet = NormalizeSbomSet(value); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? AdvisoryCursor + { + get => _advisoryCursor; + init => _advisoryCursor = Validation.NormalizeTimestamp(value); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? VexCursor + { + get => _vexCursor; + init => _vexCursor = Validation.NormalizeTimestamp(value); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public IReadOnlyDictionary Environment + { + get => _environment; + init => _environment = NormalizeEnvironment(value); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool CaptureExplain { get; init; } + + private ImmutableArray _sbomSet = ImmutableArray.Empty; + private DateTimeOffset? _advisoryCursor; + private DateTimeOffset? _vexCursor; + private IReadOnlyDictionary _environment = ImmutableSortedDictionary.Empty; + + private static ImmutableArray NormalizeSbomSet(IEnumerable? values) + => Validation.NormalizeStringSet(values, nameof(SbomSet)); + + private static ImmutableArray NormalizeSbomSet(ImmutableArray values) + => values.IsDefaultOrEmpty ? ImmutableArray.Empty : NormalizeSbomSet(values.AsEnumerable()); + + private static IReadOnlyDictionary NormalizeEnvironment(IEnumerable>? entries) + { + if (entries is null) + { + return ImmutableSortedDictionary.Empty; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var entry in entries) + { + var key = Validation.TrimToNull(entry.Key); + if (key is null) + { + continue; + } + + var normalizedKey = key.ToLowerInvariant(); + var element = entry.Value switch + { + JsonElement jsonElement => jsonElement.Clone(), + JsonDocument jsonDocument => jsonDocument.RootElement.Clone(), + string text => JsonSerializer.SerializeToElement(text).Clone(), + bool boolean => JsonSerializer.SerializeToElement(boolean).Clone(), + int integer => JsonSerializer.SerializeToElement(integer).Clone(), + long longValue => JsonSerializer.SerializeToElement(longValue).Clone(), + double doubleValue => JsonSerializer.SerializeToElement(doubleValue).Clone(), + decimal decimalValue => JsonSerializer.SerializeToElement(decimalValue).Clone(), + null => JsonSerializer.SerializeToElement(null).Clone(), + _ => JsonSerializer.SerializeToElement(entry.Value, entry.Value.GetType()).Clone(), + }; + + builder[normalizedKey] = element; + } + + return builder.ToImmutable(); + } + + private static IReadOnlyDictionary NormalizeEnvironment(IReadOnlyDictionary? environment) + { + if (environment is null || environment.Count == 0) + { + return ImmutableSortedDictionary.Empty; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var entry in environment) + { + var key = Validation.TrimToNull(entry.Key); + if (key is null) + { + continue; + } + + builder[key.ToLowerInvariant()] = entry.Value.Clone(); + } + + return builder.ToImmutable(); + } +} + +/// +/// Stored status for a policy run (policy_runs collection). +/// +public sealed record PolicyRunStatus +{ + public PolicyRunStatus( + string runId, + string tenantId, + string policyId, + int policyVersion, + PolicyRunMode mode, + PolicyRunExecutionStatus status, + PolicyRunPriority priority, + DateTimeOffset queuedAt, + PolicyRunStats? stats = null, + PolicyRunInputs? inputs = null, + DateTimeOffset? startedAt = null, + DateTimeOffset? finishedAt = null, + string? determinismHash = null, + string? errorCode = null, + string? error = null, + int attempts = 0, + string? traceId = null, + string? explainUri = null, + ImmutableSortedDictionary? metadata = null, + string? schemaVersion = null) + : this( + runId, + tenantId, + policyId, + policyVersion, + mode, + status, + priority, + Validation.NormalizeTimestamp(queuedAt), + Validation.NormalizeTimestamp(startedAt), + Validation.NormalizeTimestamp(finishedAt), + stats ?? PolicyRunStats.Empty, + inputs ?? PolicyRunInputs.Empty, + determinismHash, + Validation.TrimToNull(errorCode), + Validation.TrimToNull(error), + attempts, + Validation.TrimToNull(traceId), + Validation.TrimToNull(explainUri), + metadata ?? ImmutableSortedDictionary.Empty, + schemaVersion) + { + } + + [JsonConstructor] + public PolicyRunStatus( + string runId, + string tenantId, + string policyId, + int policyVersion, + PolicyRunMode mode, + PolicyRunExecutionStatus status, + PolicyRunPriority priority, + DateTimeOffset queuedAt, + DateTimeOffset? startedAt, + DateTimeOffset? finishedAt, + PolicyRunStats stats, + PolicyRunInputs inputs, + string? determinismHash, + string? errorCode, + string? error, + int attempts, + string? traceId, + string? explainUri, + ImmutableSortedDictionary metadata, + string? schemaVersion = null) + { + SchemaVersion = SchedulerSchemaVersions.EnsurePolicyRunStatus(schemaVersion); + RunId = Validation.EnsureId(runId, nameof(runId)); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + PolicyId = Validation.EnsureSimpleIdentifier(policyId, nameof(policyId)); + if (policyVersion <= 0) + { + throw new ArgumentOutOfRangeException(nameof(policyVersion), policyVersion, "Policy version must be positive."); + } + + PolicyVersion = policyVersion; + Mode = mode; + Status = status; + Priority = priority; + QueuedAt = Validation.NormalizeTimestamp(queuedAt); + StartedAt = Validation.NormalizeTimestamp(startedAt); + FinishedAt = Validation.NormalizeTimestamp(finishedAt); + Stats = stats ?? PolicyRunStats.Empty; + Inputs = inputs ?? PolicyRunInputs.Empty; + DeterminismHash = Validation.TrimToNull(determinismHash); + ErrorCode = Validation.TrimToNull(errorCode); + Error = Validation.TrimToNull(error); + Attempts = attempts < 0 + ? throw new ArgumentOutOfRangeException(nameof(attempts), attempts, "Attempts must be non-negative.") + : attempts; + TraceId = Validation.TrimToNull(traceId); + ExplainUri = Validation.TrimToNull(explainUri); + Metadata = (metadata ?? ImmutableSortedDictionary.Empty) + .Select(static pair => new KeyValuePair( + Validation.TrimToNull(pair.Key)?.ToLowerInvariant() ?? string.Empty, + Validation.TrimToNull(pair.Value) ?? string.Empty)) + .Where(static pair => !string.IsNullOrEmpty(pair.Key) && !string.IsNullOrEmpty(pair.Value)) + .DistinctBy(static pair => pair.Key, StringComparer.Ordinal) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); + } + + public string SchemaVersion { get; } + + public string RunId { get; } + + public string TenantId { get; } + + public string PolicyId { get; } + + public int PolicyVersion { get; } + + public PolicyRunMode Mode { get; } + + public PolicyRunExecutionStatus Status { get; init; } + + public PolicyRunPriority Priority { get; init; } + + public DateTimeOffset QueuedAt { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? StartedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? FinishedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DeterminismHash { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ErrorCode { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int Attempts { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TraceId { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ExplainUri { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Metadata { get; init; } = ImmutableSortedDictionary.Empty; + + public PolicyRunStats Stats { get; init; } = PolicyRunStats.Empty; + + public PolicyRunInputs Inputs { get; init; } = PolicyRunInputs.Empty; +} + +/// +/// Aggregated metrics captured for a policy run. +/// +public sealed record PolicyRunStats +{ + public static PolicyRunStats Empty { get; } = new(); + + public PolicyRunStats( + int components = 0, + int rulesFired = 0, + int findingsWritten = 0, + int vexOverrides = 0, + int quieted = 0, + int suppressed = 0, + double? durationSeconds = null) + { + Components = Validation.EnsureNonNegative(components, nameof(components)); + RulesFired = Validation.EnsureNonNegative(rulesFired, nameof(rulesFired)); + FindingsWritten = Validation.EnsureNonNegative(findingsWritten, nameof(findingsWritten)); + VexOverrides = Validation.EnsureNonNegative(vexOverrides, nameof(vexOverrides)); + Quieted = Validation.EnsureNonNegative(quieted, nameof(quieted)); + Suppressed = Validation.EnsureNonNegative(suppressed, nameof(suppressed)); + DurationSeconds = durationSeconds is { } seconds && seconds < 0 + ? throw new ArgumentOutOfRangeException(nameof(durationSeconds), durationSeconds, "Duration must be non-negative.") + : durationSeconds; + } + + public int Components { get; } = 0; + + public int RulesFired { get; } = 0; + + public int FindingsWritten { get; } = 0; + + public int VexOverrides { get; } = 0; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int Quieted { get; } = 0; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int Suppressed { get; } = 0; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? DurationSeconds { get; } +} + +/// +/// Summary payload returned by simulations and run diffs. +/// +public sealed record PolicyDiffSummary +{ + public PolicyDiffSummary( + int added, + int removed, + int unchanged, + IEnumerable>? bySeverity = null, + IEnumerable? ruleHits = null, + string? schemaVersion = null) + : this( + Validation.EnsureNonNegative(added, nameof(added)), + Validation.EnsureNonNegative(removed, nameof(removed)), + Validation.EnsureNonNegative(unchanged, nameof(unchanged)), + NormalizeSeverity(bySeverity), + NormalizeRuleHits(ruleHits), + schemaVersion) + { + } + + [JsonConstructor] + public PolicyDiffSummary( + int added, + int removed, + int unchanged, + ImmutableSortedDictionary bySeverity, + ImmutableArray ruleHits, + string? schemaVersion = null) + { + Added = Validation.EnsureNonNegative(added, nameof(added)); + Removed = Validation.EnsureNonNegative(removed, nameof(removed)); + Unchanged = Validation.EnsureNonNegative(unchanged, nameof(unchanged)); + BySeverity = NormalizeSeverity(bySeverity); + RuleHits = ruleHits.IsDefault ? ImmutableArray.Empty : ruleHits; + SchemaVersion = SchedulerSchemaVersions.EnsurePolicyDiffSummary(schemaVersion); + } + + public string SchemaVersion { get; } + + public int Added { get; } + + public int Removed { get; } + + public int Unchanged { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary BySeverity { get; } = ImmutableSortedDictionary.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray RuleHits { get; } = ImmutableArray.Empty; + + private static ImmutableSortedDictionary NormalizeSeverity(IEnumerable>? buckets) + { + if (buckets is null) + { + return ImmutableSortedDictionary.Empty; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var bucket in buckets) + { + var key = Validation.TrimToNull(bucket.Key); + if (key is null) + { + continue; + } + + var normalizedKey = char.ToUpperInvariant(key[0]) + key[1..].ToLowerInvariant(); + builder[normalizedKey] = bucket.Value ?? PolicyDiffSeverityDelta.Empty; + } + + return builder.ToImmutable(); + } + + private static ImmutableArray NormalizeRuleHits(IEnumerable? ruleHits) + { + if (ruleHits is null) + { + return ImmutableArray.Empty; + } + + return ruleHits + .Where(static hit => hit is not null) + .Select(static hit => hit!) + .OrderBy(static hit => hit.RuleId, StringComparer.Ordinal) + .ThenBy(static hit => hit.RuleName, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +/// +/// Delta counts for a single severity bucket. +/// +public sealed record PolicyDiffSeverityDelta +{ + public static PolicyDiffSeverityDelta Empty { get; } = new(); + + public PolicyDiffSeverityDelta(int up = 0, int down = 0) + { + Up = Validation.EnsureNonNegative(up, nameof(up)); + Down = Validation.EnsureNonNegative(down, nameof(down)); + } + + public int Up { get; } = 0; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int Down { get; } = 0; +} + +/// +/// Delta counts per rule for simulation reporting. +/// +public sealed record PolicyDiffRuleDelta +{ + public PolicyDiffRuleDelta(string ruleId, string ruleName, int up = 0, int down = 0) + { + RuleId = Validation.EnsureSimpleIdentifier(ruleId, nameof(ruleId)); + RuleName = Validation.EnsureName(ruleName, nameof(ruleName)); + Up = Validation.EnsureNonNegative(up, nameof(up)); + Down = Validation.EnsureNonNegative(down, nameof(down)); + } + + public string RuleId { get; } + + public string RuleName { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int Up { get; } = 0; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public int Down { get; } = 0; +} + +/// +/// Canonical explain trace for a policy finding. +/// +public sealed record PolicyExplainTrace +{ + public PolicyExplainTrace( + string findingId, + string policyId, + int policyVersion, + string tenantId, + string runId, + PolicyExplainVerdict verdict, + DateTimeOffset evaluatedAt, + IEnumerable? ruleChain = null, + IEnumerable? evidence = null, + IEnumerable? vexImpacts = null, + IEnumerable? history = null, + ImmutableSortedDictionary? metadata = null, + string? schemaVersion = null) + : this( + findingId, + policyId, + policyVersion, + tenantId, + runId, + Validation.NormalizeTimestamp(evaluatedAt), + verdict, + NormalizeRuleChain(ruleChain), + NormalizeEvidence(evidence), + NormalizeVexImpacts(vexImpacts), + NormalizeHistory(history), + metadata ?? ImmutableSortedDictionary.Empty, + schemaVersion) + { + } + + [JsonConstructor] + public PolicyExplainTrace( + string findingId, + string policyId, + int policyVersion, + string tenantId, + string runId, + DateTimeOffset evaluatedAt, + PolicyExplainVerdict verdict, + ImmutableArray ruleChain, + ImmutableArray evidence, + ImmutableArray vexImpacts, + ImmutableArray history, + ImmutableSortedDictionary metadata, + string? schemaVersion = null) + { + SchemaVersion = SchedulerSchemaVersions.EnsurePolicyExplainTrace(schemaVersion); + FindingId = Validation.EnsureSimpleIdentifier(findingId, nameof(findingId)); + PolicyId = Validation.EnsureSimpleIdentifier(policyId, nameof(policyId)); + if (policyVersion <= 0) + { + throw new ArgumentOutOfRangeException(nameof(policyVersion), policyVersion, "Policy version must be positive."); + } + + PolicyVersion = policyVersion; + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + RunId = Validation.EnsureId(runId, nameof(runId)); + EvaluatedAt = Validation.NormalizeTimestamp(evaluatedAt); + Verdict = verdict ?? throw new ArgumentNullException(nameof(verdict)); + RuleChain = ruleChain.IsDefault ? ImmutableArray.Empty : ruleChain; + Evidence = evidence.IsDefault ? ImmutableArray.Empty : evidence; + VexImpacts = vexImpacts.IsDefault ? ImmutableArray.Empty : vexImpacts; + History = history.IsDefault ? ImmutableArray.Empty : history; + Metadata = (metadata ?? ImmutableSortedDictionary.Empty) + .Select(static pair => new KeyValuePair( + Validation.TrimToNull(pair.Key)?.ToLowerInvariant() ?? string.Empty, + Validation.TrimToNull(pair.Value) ?? string.Empty)) + .Where(static pair => !string.IsNullOrEmpty(pair.Key) && !string.IsNullOrEmpty(pair.Value)) + .DistinctBy(static pair => pair.Key, StringComparer.Ordinal) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); + } + + public string SchemaVersion { get; } + + public string FindingId { get; } + + public string PolicyId { get; } + + public int PolicyVersion { get; } + + public string TenantId { get; } + + public string RunId { get; } + + public DateTimeOffset EvaluatedAt { get; } + + public PolicyExplainVerdict Verdict { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray RuleChain { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Evidence { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray VexImpacts { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray History { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Metadata { get; } = ImmutableSortedDictionary.Empty; + + private static ImmutableArray NormalizeRuleChain(IEnumerable? rules) + { + if (rules is null) + { + return ImmutableArray.Empty; + } + + return rules + .Where(static rule => rule is not null) + .Select(static rule => rule!) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeEvidence(IEnumerable? evidence) + { + if (evidence is null) + { + return ImmutableArray.Empty; + } + + return evidence + .Where(static item => item is not null) + .Select(static item => item!) + .OrderBy(static item => item.Type, StringComparer.Ordinal) + .ThenBy(static item => item.Reference, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeVexImpacts(IEnumerable? impacts) + { + if (impacts is null) + { + return ImmutableArray.Empty; + } + + return impacts + .Where(static impact => impact is not null) + .Select(static impact => impact!) + .OrderBy(static impact => impact.StatementId, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeHistory(IEnumerable? history) + { + if (history is null) + { + return ImmutableArray.Empty; + } + + return history + .Where(static entry => entry is not null) + .Select(static entry => entry!) + .OrderBy(static entry => entry.OccurredAt) + .ToImmutableArray(); + } +} + +/// +/// Verdict metadata for explain traces. +/// +public sealed record PolicyExplainVerdict +{ + public PolicyExplainVerdict( + PolicyVerdictStatus status, + SeverityRank? severity = null, + bool quiet = false, + double? score = null, + string? rationale = null) + { + Status = status; + Severity = severity; + Quiet = quiet; + Score = score; + Rationale = Validation.TrimToNull(rationale); + } + + public PolicyVerdictStatus Status { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public SeverityRank? Severity { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool Quiet { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? Score { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Rationale { get; } +} + +/// +/// Rule evaluation entry captured in explain traces. +/// +public sealed record PolicyExplainRule +{ + public PolicyExplainRule( + string ruleId, + string ruleName, + string action, + string decision, + double score, + string? condition = null) + { + RuleId = Validation.EnsureSimpleIdentifier(ruleId, nameof(ruleId)); + RuleName = Validation.EnsureName(ruleName, nameof(ruleName)); + Action = Validation.TrimToNull(action) ?? throw new ArgumentNullException(nameof(action)); + Decision = Validation.TrimToNull(decision) ?? throw new ArgumentNullException(nameof(decision)); + Score = score; + Condition = Validation.TrimToNull(condition); + } + + public string RuleId { get; } + + public string RuleName { get; } + + public string Action { get; } + + public string Decision { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public double Score { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Condition { get; } +} + +/// +/// Evidence entry considered during policy evaluation. +/// +public sealed record PolicyExplainEvidence +{ + public PolicyExplainEvidence( + string type, + string reference, + string source, + string status, + double weight = 0, + string? justification = null, + ImmutableSortedDictionary? metadata = null) + { + Type = Validation.TrimToNull(type) ?? throw new ArgumentNullException(nameof(type)); + Reference = Validation.TrimToNull(reference) ?? throw new ArgumentNullException(nameof(reference)); + Source = Validation.TrimToNull(source) ?? throw new ArgumentNullException(nameof(source)); + Status = Validation.TrimToNull(status) ?? throw new ArgumentNullException(nameof(status)); + Weight = weight; + Justification = Validation.TrimToNull(justification); + Metadata = (metadata ?? ImmutableSortedDictionary.Empty) + .Select(static pair => new KeyValuePair( + Validation.TrimToNull(pair.Key)?.ToLowerInvariant() ?? string.Empty, + Validation.TrimToNull(pair.Value) ?? string.Empty)) + .Where(static pair => !string.IsNullOrEmpty(pair.Key) && !string.IsNullOrEmpty(pair.Value)) + .DistinctBy(static pair => pair.Key, StringComparer.Ordinal) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToImmutableSortedDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); + } + + public string Type { get; } + + public string Reference { get; } + + public string Source { get; } + + public string Status { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public double Weight { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Justification { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Metadata { get; } = ImmutableSortedDictionary.Empty; +} + +/// +/// VEX statement impact summary captured in explain traces. +/// +public sealed record PolicyExplainVexImpact +{ + public PolicyExplainVexImpact( + string statementId, + string provider, + string status, + bool accepted, + string? justification = null, + string? confidence = null) + { + StatementId = Validation.TrimToNull(statementId) ?? throw new ArgumentNullException(nameof(statementId)); + Provider = Validation.TrimToNull(provider) ?? throw new ArgumentNullException(nameof(provider)); + Status = Validation.TrimToNull(status) ?? throw new ArgumentNullException(nameof(status)); + Accepted = accepted; + Justification = Validation.TrimToNull(justification); + Confidence = Validation.TrimToNull(confidence); + } + + public string StatementId { get; } + + public string Provider { get; } + + public string Status { get; } + + public bool Accepted { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Justification { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Confidence { get; } +} + +/// +/// History entry for a finding's policy lifecycle. +/// +public sealed record PolicyExplainHistoryEvent +{ + public PolicyExplainHistoryEvent( + string status, + DateTimeOffset occurredAt, + string? actor = null, + string? note = null) + { + Status = Validation.TrimToNull(status) ?? throw new ArgumentNullException(nameof(status)); + OccurredAt = Validation.NormalizeTimestamp(occurredAt); + Actor = Validation.TrimToNull(actor); + Note = Validation.TrimToNull(note); + } + + public string Status { get; } + + public DateTimeOffset OccurredAt { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Actor { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Note { get; } +} diff --git a/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs b/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs index 2a3f725b..4da2134b 100644 --- a/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs +++ b/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs @@ -3,24 +3,48 @@ namespace StellaOps.Scheduler.Models; /// /// Canonical schema version identifiers for scheduler documents. /// -public static class SchedulerSchemaVersions -{ - public const string Schedule = "scheduler.schedule@1"; - public const string Run = "scheduler.run@1"; - public const string ImpactSet = "scheduler.impact-set@1"; - public const string ScheduleLegacy0 = "scheduler.schedule@0"; - public const string RunLegacy0 = "scheduler.run@0"; - public const string ImpactSetLegacy0 = "scheduler.impact-set@0"; +public static class SchedulerSchemaVersions +{ + public const string Schedule = "scheduler.schedule@1"; + public const string Run = "scheduler.run@1"; + public const string ImpactSet = "scheduler.impact-set@1"; + public const string GraphBuildJob = "scheduler.graph-build-job@1"; + public const string GraphOverlayJob = "scheduler.graph-overlay-job@1"; + public const string PolicyRunRequest = "scheduler.policy-run-request@1"; + public const string PolicyRunStatus = "scheduler.policy-run-status@1"; + public const string PolicyDiffSummary = "scheduler.policy-diff-summary@1"; + public const string PolicyExplainTrace = "scheduler.policy-explain-trace@1"; + public const string ScheduleLegacy0 = "scheduler.schedule@0"; + public const string RunLegacy0 = "scheduler.run@0"; + public const string ImpactSetLegacy0 = "scheduler.impact-set@0"; + + public static string EnsureSchedule(string? value) + => Normalize(value, Schedule); - public static string EnsureSchedule(string? value) - => Normalize(value, Schedule); - - public static string EnsureRun(string? value) - => Normalize(value, Run); - - public static string EnsureImpactSet(string? value) - => Normalize(value, ImpactSet); - - private static string Normalize(string? value, string fallback) - => string.IsNullOrWhiteSpace(value) ? fallback : value.Trim(); -} + public static string EnsureRun(string? value) + => Normalize(value, Run); + + public static string EnsureImpactSet(string? value) + => Normalize(value, ImpactSet); + + public static string EnsureGraphBuildJob(string? value) + => Normalize(value, GraphBuildJob); + + public static string EnsureGraphOverlayJob(string? value) + => Normalize(value, GraphOverlayJob); + + public static string EnsurePolicyRunRequest(string? value) + => Normalize(value, PolicyRunRequest); + + public static string EnsurePolicyRunStatus(string? value) + => Normalize(value, PolicyRunStatus); + + public static string EnsurePolicyDiffSummary(string? value) + => Normalize(value, PolicyDiffSummary); + + public static string EnsurePolicyExplainTrace(string? value) + => Normalize(value, PolicyExplainTrace); + + private static string Normalize(string? value, string fallback) + => string.IsNullOrWhiteSpace(value) ? fallback : value.Trim(); +} diff --git a/src/StellaOps.Scheduler.Models/TASKS.md b/src/StellaOps.Scheduler.Models/TASKS.md index eaeb4909..3d74741e 100644 --- a/src/StellaOps.Scheduler.Models/TASKS.md +++ b/src/StellaOps.Scheduler.Models/TASKS.md @@ -10,12 +10,12 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SCHED-MODELS-20-001 | TODO | Scheduler Models Guild, Policy Guild | POLICY-ENGINE-20-000 | Define DTOs/schemas for policy runs, diffs, and explain traces (`PolicyRunRequest`, `PolicyRunStatus`, `PolicyDiffSummary`). | DTOs serialize deterministically; schema samples committed; validation helpers added. | +| SCHED-MODELS-20-001 | DONE (2025-10-26) | Scheduler Models Guild, Policy Guild | POLICY-ENGINE-20-000 | Define DTOs/schemas for policy runs, diffs, and explain traces (`PolicyRunRequest`, `PolicyRunStatus`, `PolicyDiffSummary`). | DTOs serialize deterministically; schema samples committed; validation helpers added. | | SCHED-MODELS-20-002 | TODO | Scheduler Models Guild | SCHED-MODELS-20-001 | Extend scheduler schema docs to include policy run lifecycle, environment metadata, and diff payloads. | Docs updated with compliance checklist; samples validated against JSON schema; consumers notified. | ## Graph Explorer v1 (Sprint 21) | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SCHED-MODELS-21-001 | TODO | Scheduler Models Guild, Cartographer Guild | CARTO-GRAPH-21-007 | Define job DTOs for graph builds/overlay refresh (`GraphBuildJob`, `GraphOverlayJob`) with deterministic serialization and status enums. | DTOs serialized deterministically; schema snippets documented; tests cover transitions. | -| SCHED-MODELS-21-002 | TODO | Scheduler Models Guild | SCHED-MODELS-21-001 | Publish schema docs/sample payloads for graph jobs and overlay events for downstream workers/UI. | Docs updated with compliance checklist; samples validated; notifications sent to guilds. | +| SCHED-MODELS-21-001 | DONE (2025-10-26) | Scheduler Models Guild, Cartographer Guild | CARTO-GRAPH-21-007 | Define job DTOs for graph builds/overlay refresh (`GraphBuildJob`, `GraphOverlayJob`) with deterministic serialization and status enums. | DTOs serialized deterministically; schema snippets documented in `src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md`; tests cover transitions. | +| SCHED-MODELS-21-002 | DONE (2025-10-26) | Scheduler Models Guild | SCHED-MODELS-21-001 | Publish schema docs/sample payloads for graph jobs and overlay events for downstream workers/UI. | Docs updated with compliance checklist; samples validated; notifications sent to guilds. | diff --git a/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md new file mode 100644 index 00000000..cc2ce2ce --- /dev/null +++ b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md @@ -0,0 +1,148 @@ +# SCHED-MODELS-20-001 — Policy Engine Run DTOs + +> Status: 2025-10-26 — **Complete** + +Defines the scheduler contracts that Policy Engine (Epic 2) relies on for orchestration, simulation, and explainability. DTOs serialize with `CanonicalJsonSerializer` to guarantee deterministic ordering, enabling replay and signed artefacts. + +## PolicyRunRequest — `scheduler.policy-run-request@1` + +Posted by CLI/UI or the orchestrator to enqueue a run. Canonical sample lives at `samples/api/scheduler/policy-run-request.json`. + +```jsonc +{ + "schemaVersion": "scheduler.policy-run-request@1", + "tenantId": "default", + "policyId": "P-7", + "policyVersion": 4, + "mode": "incremental", // full | incremental | simulate + "priority": "normal", // normal | high | emergency + "runId": "run:P-7:2025-10-26:auto", // optional idempotency key + "queuedAt": "2025-10-26T14:05:00+00:00", + "requestedBy": "user:cli", + "correlationId": "req-...", + "metadata": {"source": "stella policy run", "trigger": "cli"}, + "inputs": { + "sbomSet": ["sbom:S-318", "sbom:S-42"], // sorted uniques + "advisoryCursor": "2025-10-26T13:59:00+00:00", + "vexCursor": "2025-10-26T13:58:30+00:00", + "environment": {"exposure": "internet", "sealed": false}, + "captureExplain": true + } +} +``` + +* Environment values accept any JSON primitive/object; keys normalise to lowercase for deterministic hashing. +* `metadata` is optional contextual breadcrumbs (lowercased keys). Use it for orchestrator provenance or offline bundle identifiers. + +## PolicyRunStatus — `scheduler.policy-run-status@1` + +Captured in `policy_runs` collection and returned by run status APIs. Sample: `samples/api/scheduler/policy-run-status.json`. + +```jsonc +{ + "schemaVersion": "scheduler.policy-run-status@1", + "runId": "run:P-7:2025-10-26:auto", + "tenantId": "default", + "policyId": "P-7", + "policyVersion": 4, + "mode": "incremental", + "status": "succeeded", // queued|running|succeeded|failed|canceled|replay_pending + "priority": "normal", + "queuedAt": "2025-10-26T14:05:00+00:00", + "startedAt": "2025-10-26T14:05:11+00:00", + "finishedAt": "2025-10-26T14:06:01+00:00", + "determinismHash": "sha256:...", // optional until run completes + "traceId": "01HE0BJX5S4T9YCN6ZT0", + "metadata": {"orchestrator": "scheduler", "sbombatchhash": "sha256:..."}, + "stats": { + "components": 1742, + "rulesFired": 68023, + "findingsWritten": 4321, + "vexOverrides": 210, + "quieted": 12, + "durationSeconds": 50.8 + }, + "inputs": { ... } // same schema as request +} +``` + +* `determinismHash` must be a `sha256:` digest combining ordered input digests + policy digest. +* `attempts` (not shown) increases per retry. +* Error responses populate `errorCode` (`ERR_POL_00x`) and `error` message; omitted when successful. + +## PolicyDiffSummary — `scheduler.policy-diff-summary@1` + +Returned by simulation APIs; referenced by CLI/UI diff visualisations. Sample: `samples/api/scheduler/policy-diff-summary.json`. + +```jsonc +{ + "schemaVersion": "scheduler.policy-diff-summary@1", + "added": 12, + "removed": 8, + "unchanged": 657, + "bySeverity": { + "critical": {"up": 1}, + "high": {"up": 3, "down": 4}, + "medium": {"up": 2, "down": 1} + }, + "ruleHits": [ + {"ruleId": "rule-block-critical", "ruleName": "Block Critical Findings", "up": 1}, + {"ruleId": "rule-quiet-low", "ruleName": "Quiet Low Risk", "down": 2} + ] +} +``` + +* Severity bucket keys normalise to camelCase for JSON determinism across CLI/UI consumers. +* Zero-valued counts (`down`/`up`) are omitted to keep payloads compact. +* `ruleHits` sorts by `ruleId` to keep diff heatmaps deterministic. + +## PolicyExplainTrace — `scheduler.policy-explain-trace@1` + +Canonical explain tree embedded in findings explainers and exported bundles. Sample: `samples/api/scheduler/policy-explain-trace.json`. + +```jsonc +{ + "schemaVersion": "scheduler.policy-explain-trace@1", + "findingId": "finding:sbom:S-42/pkg:npm/lodash@4.17.21", + "policyId": "P-7", + "policyVersion": 4, + "tenantId": "default", + "runId": "run:P-7:2025-10-26:auto", + "evaluatedAt": "2025-10-26T14:06:01+00:00", + "verdict": {"status": "blocked", "severity": "critical", "score": 19.5}, + "ruleChain": [ + {"ruleId": "rule-allow-known", "action": "allow", "decision": "skipped"}, + {"ruleId": "rule-block-critical", "action": "block", "decision": "matched", "score": 19.5} + ], + "evidence": [ + {"type": "advisory", "reference": "CVE-2025-12345", "source": "nvd", "status": "affected", "weight": 1, "metadata": {}}, + {"type": "vex", "reference": "vex:ghsa-2025-0001", "source": "vendor", "status": "not_affected", "weight": 0.5} + ], + "vexImpacts": [ + {"statementId": "vex:ghsa-2025-0001", "provider": "vendor", "status": "not_affected", "accepted": true} + ], + "history": [ + {"status": "blocked", "occurredAt": "2025-10-26T14:06:01+00:00", "actor": "policy-engine"} + ], + "metadata": {"componentpurl": "pkg:npm/lodash@4.17.21", "sbomid": "sbom:S-42", "traceid": "01HE0BJX5S4T9YCN6ZT0"} +} +``` + +* Rule chain preserves execution order; evidence & VEX arrays sort for deterministic outputs. +* Evidence metadata is always emitted (empty object when no attributes) so clients can merge annotations deterministically. +* Metadata keys lower-case for consistent lookups (`componentpurl`, `traceid`, etc.). +* `verdict.status` uses `passed|warned|blocked|quieted|ignored` reflecting final policy decision. + +## Compliance Checklist + +| Item | Owner | Status | Notes | +| --- | --- | --- | --- | +| Canonical samples committed (`policy-run-request|status|diff-summary|explain-trace`) | Scheduler Models Guild | ☑ 2025-10-26 | Round-trip tests enforce schema stability. | +| DTOs documented here and linked from `/docs/policy/runs.md` checklist | Scheduler Models Guild | ☑ 2025-10-26 | Added Run DTO schema section. | +| Serializer ensures deterministic ordering for new types | Scheduler Models Guild | ☑ 2025-10-26 | `CanonicalJsonSerializer` updated with property order + converters. | +| Tests cover DTO validation and sample fixtures | Scheduler Models Guild | ☑ 2025-10-26 | `PolicyRunModelsTests` + extended `SamplePayloadTests`. | +| Scheduler guilds notified (Models, Worker, WebService) | Scheduler Models Guild | ☑ 2025-10-26 | Posted in `#scheduler-guild` with sample links. | + +--- + +*Last updated: 2025-10-26.* diff --git a/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md new file mode 100644 index 00000000..abd927f4 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-21-001-GRAPH-JOBS.md @@ -0,0 +1,107 @@ +# SCHED-MODELS-21-001 — Graph Job DTOs + +> Status: 2025-10-26 — **Complete** + +Defines the scheduler-facing contracts for Cartographer orchestration. Both DTOs serialize with `CanonicalJsonSerializer` and share the `GraphJobStatus` lifecycle guarded by `GraphJobStateMachine`. + +## GraphBuildJob — `scheduler.graph-build-job@1` + +```jsonc +{ + "schemaVersion": "scheduler.graph-build-job@1", + "id": "gbj_...", + "tenantId": "tenant-id", + "sbomId": "sbom-id", + "sbomVersionId": "sbom-version-id", + "sbomDigest": "sha256:<64-hex>", + "graphSnapshotId": "graph-snapshot-id?", // optional until Cartographer returns id + "status": "pending|queued|running|completed|failed|cancelled", + "trigger": "sbom-version|backfill|manual", + "attempts": 0, + "cartographerJobId": "external-id?", // optional identifier returned by Cartographer + "correlationId": "evt-...", // optional event correlation key + "createdAt": "2025-10-26T12:00:00+00:00", + "startedAt": "2025-10-26T12:00:05+00:00?", + "completedAt": "2025-10-26T12:00:35+00:00?", + "error": "cartographer timeout?", // populated only for failed state + "metadata": { // extra provenance (sorted, case-insensitive keys) + "sbomEventId": "sbom_evt_123" + } +} +``` + +* `sbomDigest` must be a lowercase `sha256:` string. +* `attempts` is monotonic across retries; `GraphJobStateMachine.EnsureTransition` enforces non-decreasing values and timestamps. +* Terminal states (`completed|failed|cancelled`) require `completedAt` to be set; failures require `error`. + +## GraphOverlayJob — `scheduler.graph-overlay-job@1` + +```jsonc +{ + "schemaVersion": "scheduler.graph-overlay-job@1", + "id": "goj_...", + "tenantId": "tenant-id", + "graphSnapshotId": "graph-snapshot-id", + "buildJobId": "gbj_...?", + "overlayKind": "policy|advisory|vex", + "overlayKey": "policy@2025-10-01", + "subjects": [ + "artifact/service-api", + "artifact/service-worker" + ], + "status": "pending|queued|running|completed|failed|cancelled", + "trigger": "policy|advisory|vex|sbom-version|manual", + "attempts": 0, + "correlationId": "policy_run_321?", + "createdAt": "2025-10-26T12:05:00+00:00", + "startedAt": "2025-10-26T12:05:05+00:00?", + "completedAt": "2025-10-26T12:05:15+00:00?", + "error": "overlay build failed?", + "metadata": { + "policyRunId": "policy_run_321" + } +} +``` + +* `overlayKey` is free-form but trimmed; `subjects` are deduplicated and lexicographically ordered. +* `GraphOverlayJobTrigger` strings (`policy`, `advisory`, `vex`, `sbom-version`, `manual`) align with upstream events (Policy Engine, Conseiller, Excititor, SBOM Service, or manual enqueue). +* State invariants mirror build jobs: timestamps advance monotonically, terminal states require `completedAt`, failures require `error`. + +## Status & trigger matrix + +| Enum | JSON values | +| --- | --- | +| `GraphJobStatus` | `pending`, `queued`, `running`, `completed`, `failed`, `cancelled` | +| `GraphBuildJobTrigger` | `sbom-version`, `backfill`, `manual` | +| `GraphOverlayJobTrigger` | `policy`, `advisory`, `vex`, `sbom-version`, `manual` | +| `GraphOverlayKind` | `policy`, `advisory`, `vex` | + +`GraphJobStateMachine` exposes `CanTransition` and `EnsureTransition(...)` helpers to keep scheduler workers deterministic and to centralize validation logic. Callers must provide an error message when moving to `failed`; other states clear the error automatically. + +--- + +## Published samples + +- `samples/api/scheduler/graph-build-job.json` – canonical Cartographer build request snapshot (status `running`, one retry). +- `samples/api/scheduler/graph-overlay-job.json` – queued policy overlay job with deduplicated `subjects`. +- `docs/events/samples/scheduler.graph.job.completed@1.sample.json` – legacy completion event embedding the canonical job payload for downstream caches/UI. + +Tests in `StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs` validate the job fixtures against the canonical serializer. + +--- + +## Events + +Scheduler emits `scheduler.graph.job.completed@1` when a graph build or overlay job reaches `completed`, `failed`, or `cancelled`. Schema lives at `docs/events/scheduler.graph.job.completed@1.json` (legacy envelope) and the sample above illustrates the canonical payload. Downstream services should validate their consumers against the schema and budget for eventual migration to the orchestrator envelope once Cartographer hooks are promoted. + +--- + +## Compliance checklist + +| Item | Owner | Status | Notes | +| --- | --- | --- | --- | +| Canonical graph job samples committed under `samples/api/scheduler` | Scheduler Models Guild | ☑ 2025-10-26 | Round-trip tests cover both payloads. | +| Schema doc published with trigger/status matrix and sample references | Scheduler Models Guild | ☑ 2025-10-26 | This document. | +| Event schema + sample published under `docs/events/` | Scheduler Models Guild | ☑ 2025-10-26 | `scheduler.graph.job.completed@1` covers terminal job events. | +| Notify Scheduler WebService & Worker guilds about new DTO availability | Scheduler Models Guild | ☑ 2025-10-26 | Announcement posted (see `docs/updates/2025-10-26-scheduler-graph-jobs.md`). | +| Notify Cartographer Guild about expected job metadata (`graphSnapshotId`, `cartographerJobId`) | Scheduler Models Guild | ☑ 2025-10-26 | Included in Cartographer sync note (`docs/updates/2025-10-26-scheduler-graph-jobs.md`). | diff --git a/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj b/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj index bce5c247..d5994e7a 100644 --- a/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj +++ b/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj @@ -1,27 +1,27 @@ - - - net10.0 - enable - enable - false - false - - + + + net10.0 + enable + enable + false + false + + - - + + - - all - - - all - - - - - - - + + all + + + all + + + + + + + diff --git a/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj b/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj index 74b0c48b..5b94daa2 100644 --- a/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj +++ b/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj @@ -1,21 +1,21 @@ - - - net10.0 - enable - enable - + + + net10.0 + enable + enable + - - - - - - + + + + + + - - - - + + + + diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/AuditRepositoryTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/AuditRepositoryTests.cs new file mode 100644 index 00000000..6aed9f4e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/AuditRepositoryTests.cs @@ -0,0 +1,60 @@ +using System; +using System.Linq; +using System.Threading; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories; + +public sealed class AuditRepositoryTests +{ + [Fact] + public async Task InsertAndListAsync_ReturnsTenantScopedEntries() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new AuditRepository(harness.Context); + + var record1 = TestDataFactory.CreateAuditRecord("tenant-alpha", "1"); + var record2 = TestDataFactory.CreateAuditRecord("tenant-alpha", "2"); + var otherTenant = TestDataFactory.CreateAuditRecord("tenant-beta", "3"); + + await repository.InsertAsync(record1); + await repository.InsertAsync(record2); + await repository.InsertAsync(otherTenant); + + var results = await repository.ListAsync("tenant-alpha"); + Assert.Equal(2, results.Count); + Assert.DoesNotContain(results, record => record.TenantId == "tenant-beta"); + } + + [Fact] + public async Task ListAsync_AppliesFilters() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new AuditRepository(harness.Context); + + var older = TestDataFactory.CreateAuditRecord( + "tenant-alpha", + "old", + occurredAt: DateTimeOffset.UtcNow.AddMinutes(-30), + scheduleId: "sch-a"); + var newer = TestDataFactory.CreateAuditRecord( + "tenant-alpha", + "new", + occurredAt: DateTimeOffset.UtcNow, + scheduleId: "sch-a"); + + await repository.InsertAsync(older); + await repository.InsertAsync(newer); + + var options = new AuditQueryOptions + { + Since = DateTimeOffset.UtcNow.AddMinutes(-5), + ScheduleId = "sch-a", + Limit = 5 + }; + + var results = await repository.ListAsync("tenant-alpha", options); + Assert.Single(results); + Assert.Equal("audit_new", results.Single().Id); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/ImpactSnapshotRepositoryTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/ImpactSnapshotRepositoryTests.cs new file mode 100644 index 00000000..c9b7077e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/ImpactSnapshotRepositoryTests.cs @@ -0,0 +1,41 @@ +using System; +using System.Threading; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories; + +public sealed class ImpactSnapshotRepositoryTests +{ + [Fact] + public async Task UpsertAndGetAsync_RoundTripsSnapshot() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new ImpactSnapshotRepository(harness.Context); + + var snapshot = TestDataFactory.CreateImpactSet("tenant-alpha", "impact-1", DateTimeOffset.UtcNow.AddMinutes(-5)); + await repository.UpsertAsync(snapshot, cancellationToken: CancellationToken.None); + + var stored = await repository.GetBySnapshotIdAsync("impact-1", cancellationToken: CancellationToken.None); + Assert.NotNull(stored); + Assert.Equal(snapshot.SnapshotId, stored!.SnapshotId); + Assert.Equal(snapshot.Images[0].ImageDigest, stored.Images[0].ImageDigest); + } + + [Fact] + public async Task GetLatestBySelectorAsync_ReturnsMostRecent() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new ImpactSnapshotRepository(harness.Context); + + var selectorTenant = "tenant-alpha"; + var first = TestDataFactory.CreateImpactSet(selectorTenant, "impact-old", DateTimeOffset.UtcNow.AddMinutes(-10)); + var latest = TestDataFactory.CreateImpactSet(selectorTenant, "impact-new", DateTimeOffset.UtcNow); + + await repository.UpsertAsync(first); + await repository.UpsertAsync(latest); + + var resolved = await repository.GetLatestBySelectorAsync(latest.Selector); + Assert.NotNull(resolved); + Assert.Equal("impact-new", resolved!.SnapshotId); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/RunRepositoryTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/RunRepositoryTests.cs new file mode 100644 index 00000000..3178666e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/RunRepositoryTests.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories; + +public sealed class RunRepositoryTests +{ + [Fact] + public async Task InsertAndGetAsync_RoundTripsRun() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new RunRepository(harness.Context); + + var run = TestDataFactory.CreateRun("run_1", "tenant-alpha", RunState.Planning); + await repository.InsertAsync(run, cancellationToken: CancellationToken.None); + + var stored = await repository.GetAsync(run.TenantId, run.Id, cancellationToken: CancellationToken.None); + Assert.NotNull(stored); + Assert.Equal(run.State, stored!.State); + Assert.Equal(run.Trigger, stored.Trigger); + } + + [Fact] + public async Task UpdateAsync_ChangesStateAndStats() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new RunRepository(harness.Context); + + var run = TestDataFactory.CreateRun("run_update", "tenant-alpha", RunState.Planning); + await repository.InsertAsync(run); + + var updated = run with + { + State = RunState.Completed, + FinishedAt = DateTimeOffset.UtcNow, + Stats = new RunStats(candidates: 10, deduped: 10, queued: 10, completed: 10, deltas: 2) + }; + + var result = await repository.UpdateAsync(updated); + Assert.True(result); + + var stored = await repository.GetAsync(updated.TenantId, updated.Id); + Assert.NotNull(stored); + Assert.Equal(RunState.Completed, stored!.State); + Assert.Equal(10, stored.Stats.Completed); + } + + [Fact] + public async Task ListAsync_FiltersByStateAndSchedule() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new RunRepository(harness.Context); + + var run1 = TestDataFactory.CreateRun("run_state_1", "tenant-alpha", RunState.Planning, scheduleId: "sch_a"); + var run2 = TestDataFactory.CreateRun("run_state_2", "tenant-alpha", RunState.Running, scheduleId: "sch_a"); + var run3 = TestDataFactory.CreateRun("run_state_3", "tenant-alpha", RunState.Completed, scheduleId: "sch_b"); + + await repository.InsertAsync(run1); + await repository.InsertAsync(run2); + await repository.InsertAsync(run3); + + var options = new RunQueryOptions + { + ScheduleId = "sch_a", + States = new[] { RunState.Running }.ToImmutableArray(), + Limit = 10 + }; + + var results = await repository.ListAsync("tenant-alpha", options); + Assert.Single(results); + Assert.Equal("run_state_2", results.Single().Id); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/ScheduleRepositoryTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/ScheduleRepositoryTests.cs new file mode 100644 index 00000000..8309ff95 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Repositories/ScheduleRepositoryTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Threading; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Repositories; + +public sealed class ScheduleRepositoryTests +{ + [Fact] + public async Task UpsertAsync_PersistsScheduleWithCanonicalShape() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new ScheduleRepository(harness.Context); + + var schedule = TestDataFactory.CreateSchedule("sch_unit_1", "tenant-alpha"); + await repository.UpsertAsync(schedule, cancellationToken: CancellationToken.None); + + var stored = await repository.GetAsync(schedule.TenantId, schedule.Id, cancellationToken: CancellationToken.None); + Assert.NotNull(stored); + Assert.Equal(schedule.Id, stored!.Id); + Assert.Equal(schedule.Name, stored.Name); + Assert.Equal(schedule.Selection.Scope, stored.Selection.Scope); + } + + [Fact] + public async Task ListAsync_ExcludesDisabledAndDeletedByDefault() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new ScheduleRepository(harness.Context); + var tenantId = "tenant-alpha"; + + var enabled = TestDataFactory.CreateSchedule("sch_enabled", tenantId, enabled: true, name: "Enabled"); + var disabled = TestDataFactory.CreateSchedule("sch_disabled", tenantId, enabled: false, name: "Disabled"); + + await repository.UpsertAsync(enabled); + await repository.UpsertAsync(disabled); + await repository.SoftDeleteAsync(tenantId, enabled.Id, "svc_scheduler", DateTimeOffset.UtcNow); + + var results = await repository.ListAsync(tenantId); + Assert.Empty(results); + + var includeDisabled = await repository.ListAsync( + tenantId, + new ScheduleQueryOptions { IncludeDisabled = true, IncludeDeleted = true }); + + Assert.Equal(2, includeDisabled.Count); + Assert.Contains(includeDisabled, schedule => schedule.Id == enabled.Id); + Assert.Contains(includeDisabled, schedule => schedule.Id == disabled.Id); + } + + [Fact] + public async Task SoftDeleteAsync_SetsMetadataAndExcludesFromQueries() + { + using var harness = new SchedulerMongoTestHarness(); + var repository = new ScheduleRepository(harness.Context); + + var schedule = TestDataFactory.CreateSchedule("sch_delete", "tenant-beta"); + await repository.UpsertAsync(schedule); + + var deletedAt = DateTimeOffset.UtcNow; + var deleted = await repository.SoftDeleteAsync(schedule.TenantId, schedule.Id, "svc_delete", deletedAt); + Assert.True(deleted); + + var retrieved = await repository.GetAsync(schedule.TenantId, schedule.Id); + Assert.Null(retrieved); + + var includeDeleted = await repository.ListAsync( + schedule.TenantId, + new ScheduleQueryOptions { IncludeDeleted = true, IncludeDisabled = true }); + + Assert.Single(includeDeleted); + Assert.Equal("sch_delete", includeDeleted[0].Id); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/SchedulerMongoTestHarness.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/SchedulerMongoTestHarness.cs new file mode 100644 index 00000000..250870ad --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/SchedulerMongoTestHarness.cs @@ -0,0 +1,36 @@ +using System; +using System.Threading; +using Microsoft.Extensions.Logging.Abstractions; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests; + +internal sealed class SchedulerMongoTestHarness : IDisposable +{ + private readonly MongoDbRunner _runner; + + public SchedulerMongoTestHarness() + { + _runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet"); + var options = new SchedulerMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = $"scheduler_tests_{Guid.NewGuid():N}" + }; + + Context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger.Instance); + var migrations = new ISchedulerMongoMigration[] + { + new EnsureSchedulerCollectionsMigration(NullLogger.Instance), + new EnsureSchedulerIndexesMigration() + }; + var runner = new SchedulerMongoMigrationRunner(Context, migrations, NullLogger.Instance); + runner.RunAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + + public SchedulerMongoContext Context { get; } + + public void Dispose() + { + _runner.Dispose(); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Services/RunSummaryServiceTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Services/RunSummaryServiceTests.cs new file mode 100644 index 00000000..6e96a767 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Services/RunSummaryServiceTests.cs @@ -0,0 +1,116 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Services; + +public sealed class RunSummaryServiceTests : IDisposable +{ + private readonly SchedulerMongoTestHarness _harness; + private readonly RunSummaryRepository _repository; + private readonly StubTimeProvider _timeProvider; + private readonly RunSummaryService _service; + + public RunSummaryServiceTests() + { + _harness = new SchedulerMongoTestHarness(); + _repository = new RunSummaryRepository(_harness.Context); + _timeProvider = new StubTimeProvider(DateTimeOffset.Parse("2025-10-26T10:00:00Z")); + _service = new RunSummaryService(_repository, _timeProvider, NullLogger.Instance); + } + + [Fact] + public async Task ProjectAsync_FirstRunCreatesProjection() + { + var run = TestDataFactory.CreateRun("run-1", "tenant-alpha", RunState.Planning, "sch-alpha"); + + var projection = await _service.ProjectAsync(run, CancellationToken.None); + + Assert.Equal("tenant-alpha", projection.TenantId); + Assert.Equal("sch-alpha", projection.ScheduleId); + Assert.NotNull(projection.LastRun); + Assert.Equal(RunState.Planning, projection.LastRun!.State); + Assert.Equal(1, projection.Counters.Total); + Assert.Equal(1, projection.Counters.Planning); + Assert.Equal(0, projection.Counters.Completed); + Assert.Single(projection.Recent); + Assert.Equal(run.Id, projection.Recent[0].RunId); + } + + [Fact] + public async Task ProjectAsync_UpdateRunReplacesExistingEntry() + { + var createdAt = DateTimeOffset.Parse("2025-10-26T09:55:00Z"); + var run = TestDataFactory.CreateRun( + "run-update", + "tenant-alpha", + RunState.Planning, + "sch-alpha", + createdAt: createdAt, + startedAt: createdAt.AddMinutes(1)); + await _service.ProjectAsync(run, CancellationToken.None); + + var updated = run with + { + State = RunState.Completed, + StartedAt = run.StartedAt, + FinishedAt = run.CreatedAt.AddMinutes(5), + Stats = new RunStats(candidates: 10, deduped: 8, queued: 5, completed: 10, deltas: 2, newCriticals: 1) + }; + + _timeProvider.Advance(TimeSpan.FromMinutes(10)); + var projection = await _service.ProjectAsync(updated, CancellationToken.None); + + Assert.NotNull(projection.LastRun); + Assert.Equal(RunState.Completed, projection.LastRun!.State); + Assert.Equal(1, projection.Counters.Completed); + Assert.Equal(0, projection.Counters.Planning); + Assert.Single(projection.Recent); + Assert.Equal(updated.Stats.Completed, projection.LastRun!.Stats.Completed); + Assert.True(projection.UpdatedAt > run.CreatedAt); + } + + [Fact] + public async Task ProjectAsync_TrimsRecentEntriesBeyondLimit() + { + var baseTime = DateTimeOffset.Parse("2025-10-26T00:00:00Z"); + + for (var i = 0; i < 25; i++) + { + var run = TestDataFactory.CreateRun( + $"run-{i}", + "tenant-alpha", + RunState.Completed, + "sch-alpha", + stats: new RunStats(candidates: 5, deduped: 4, queued: 3, completed: 5, deltas: 1), + createdAt: baseTime.AddMinutes(i)); + + await _service.ProjectAsync(run, CancellationToken.None); + } + + var projections = await _service.ListAsync("tenant-alpha", CancellationToken.None); + Assert.Single(projections); + var projection = projections[0]; + Assert.Equal(20, projection.Recent.Length); + Assert.Equal(20, projection.Counters.Total); + Assert.Equal("run-24", projection.Recent[0].RunId); + } + + public void Dispose() + { + _harness.Dispose(); + } + + private sealed class StubTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public StubTimeProvider(DateTimeOffset initial) + => _utcNow = initial; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan delta) => _utcNow = _utcNow.Add(delta); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Services/SchedulerAuditServiceTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Services/SchedulerAuditServiceTests.cs new file mode 100644 index 00000000..54c16689 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Services/SchedulerAuditServiceTests.cs @@ -0,0 +1,82 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Services; + +public sealed class SchedulerAuditServiceTests : IDisposable +{ + private readonly SchedulerMongoTestHarness _harness; + private readonly AuditRepository _repository; + private readonly StubTimeProvider _timeProvider; + private readonly SchedulerAuditService _service; + + public SchedulerAuditServiceTests() + { + _harness = new SchedulerMongoTestHarness(); + _repository = new AuditRepository(_harness.Context); + _timeProvider = new StubTimeProvider(DateTimeOffset.Parse("2025-10-26T11:30:00Z")); + _service = new SchedulerAuditService(_repository, _timeProvider, NullLogger.Instance); + } + + [Fact] + public async Task WriteAsync_PersistsRecordWithGeneratedId() + { + var auditEvent = new SchedulerAuditEvent( + TenantId: "tenant-alpha", + Category: "scheduler", + Action: "create", + Actor: new AuditActor("user_admin", "Admin", "user"), + ScheduleId: "sch-alpha", + CorrelationId: "corr-1", + Metadata: new Dictionary + { + ["Reason"] = "initial", + }, + Message: "created schedule"); + + var record = await _service.WriteAsync(auditEvent, CancellationToken.None); + + Assert.StartsWith("audit_", record.Id, StringComparison.Ordinal); + Assert.Equal(_timeProvider.GetUtcNow(), record.OccurredAt); + + var stored = await _repository.ListAsync("tenant-alpha", new AuditQueryOptions { ScheduleId = "sch-alpha" }, session: null, CancellationToken.None); + Assert.Single(stored); + Assert.Equal(record.Id, stored[0].Id); + Assert.Equal("created schedule", stored[0].Message); + Assert.Contains(stored[0].Metadata, pair => pair.Key == "reason" && pair.Value == "initial"); + } + + [Fact] + public async Task WriteAsync_HonoursProvidedAuditId() + { + var auditEvent = new SchedulerAuditEvent( + TenantId: "tenant-alpha", + Category: "scheduler", + Action: "update", + Actor: new AuditActor("user_admin", "Admin", "user"), + ScheduleId: "sch-alpha", + AuditId: "audit_custom_1", + OccurredAt: DateTimeOffset.Parse("2025-10-26T12:00:00Z")); + + var record = await _service.WriteAsync(auditEvent, CancellationToken.None); + Assert.Equal("audit_custom_1", record.Id); + Assert.Equal(DateTimeOffset.Parse("2025-10-26T12:00:00Z"), record.OccurredAt); + } + + public void Dispose() + { + _harness.Dispose(); + } + + private sealed class StubTimeProvider : TimeProvider + { + private DateTimeOffset _utcNow; + + public StubTimeProvider(DateTimeOffset initial) + => _utcNow = initial; + + public override DateTimeOffset GetUtcNow() => _utcNow; + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Sessions/SchedulerMongoSessionFactoryTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Sessions/SchedulerMongoSessionFactoryTests.cs new file mode 100644 index 00000000..9e3da828 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Sessions/SchedulerMongoSessionFactoryTests.cs @@ -0,0 +1,35 @@ +using System.Threading; +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Sessions; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Sessions; + +public sealed class SchedulerMongoSessionFactoryTests +{ + [Fact] + public async Task StartSessionAsync_UsesCausalConsistencyByDefault() + { + using var harness = new SchedulerMongoTestHarness(); + var factory = new SchedulerMongoSessionFactory(harness.Context); + + using var session = await factory.StartSessionAsync(cancellationToken: CancellationToken.None); + Assert.True(session.Options.CausalConsistency.GetValueOrDefault()); + } + + [Fact] + public async Task StartSessionAsync_AllowsOverridingOptions() + { + using var harness = new SchedulerMongoTestHarness(); + var factory = new SchedulerMongoSessionFactory(harness.Context); + + var options = new SchedulerMongoSessionOptions + { + CausalConsistency = false, + ReadPreference = ReadPreference.PrimaryPreferred + }; + + using var session = await factory.StartSessionAsync(options); + Assert.False(session.Options.CausalConsistency.GetValueOrDefault(true)); + Assert.Equal(ReadPreference.PrimaryPreferred, session.Options.DefaultTransactionOptions?.ReadPreference); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj b/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj index 65345131..60e78365 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj @@ -1,23 +1,23 @@ - - - net10.0 - enable - enable - false - - - - - - - - - - - - - - Always - - - + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + Always + + + diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/TestDataFactory.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/TestDataFactory.cs new file mode 100644 index 00000000..2736368e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/TestDataFactory.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests; + +internal static class TestDataFactory +{ + public static Schedule CreateSchedule( + string id, + string tenantId, + bool enabled = true, + string name = "Nightly Prod") + { + var now = DateTimeOffset.UtcNow; + return new Schedule( + id, + tenantId, + name, + enabled, + "0 2 * * *", + "UTC", + ScheduleMode.AnalysisOnly, + new Selector(SelectorScope.AllImages, tenantId), + ScheduleOnlyIf.Default, + ScheduleNotify.Default, + ScheduleLimits.Default, + now, + "svc_scheduler", + now, + "svc_scheduler", + ImmutableArray.Empty, + SchedulerSchemaVersions.Schedule); + } + + public static Run CreateRun( + string id, + string tenantId, + RunState state, + string? scheduleId = null, + RunTrigger trigger = RunTrigger.Manual, + RunStats? stats = null, + DateTimeOffset? createdAt = null, + DateTimeOffset? startedAt = null) + { + var resolvedStats = stats ?? new RunStats(candidates: 10, deduped: 8, queued: 5, completed: 0, deltas: 2); + var created = createdAt ?? DateTimeOffset.UtcNow; + return new Run( + id, + tenantId, + trigger, + state, + resolvedStats, + created, + scheduleId: scheduleId, + reason: new RunReason(manualReason: "test"), + startedAt: startedAt ?? created); + } + + public static ImpactSet CreateImpactSet(string tenantId, string snapshotId, DateTimeOffset? generatedAt = null, bool usageOnly = true) + { + var selector = new Selector(SelectorScope.AllImages, tenantId); + var image = new ImpactImage( + "sha256:" + Guid.NewGuid().ToString("N"), + "registry", + "repo/app", + namespaces: new[] { "team-a" }, + tags: new[] { "prod" }, + usedByEntrypoint: true); + + return new ImpactSet( + selector, + new[] { image }, + usageOnly: usageOnly, + generatedAt ?? DateTimeOffset.UtcNow, + total: 1, + snapshotId: snapshotId, + schemaVersion: SchedulerSchemaVersions.ImpactSet); + } + + public static AuditRecord CreateAuditRecord( + string tenantId, + string idSuffix, + DateTimeOffset? occurredAt = null, + string? scheduleId = null, + string? category = null, + string? action = null) + { + return new AuditRecord( + $"audit_{idSuffix}", + tenantId, + category ?? "scheduler", + action ?? "create", + occurredAt ?? DateTimeOffset.UtcNow, + new AuditActor("user_admin", "Admin", "user"), + scheduleId: scheduleId ?? $"sch_{idSuffix}", + message: "created"); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Documents/RunSummaryDocument.cs b/src/StellaOps.Scheduler.Storage.Mongo/Documents/RunSummaryDocument.cs new file mode 100644 index 00000000..20ee8089 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Documents/RunSummaryDocument.cs @@ -0,0 +1,88 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Documents; + +[BsonIgnoreExtraElements] +internal sealed class RunSummaryDocument +{ + public string Id { get; set; } = string.Empty; + + public string TenantId { get; set; } = string.Empty; + + public string ScheduleId { get; set; } = string.Empty; + + public DateTime UpdatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UnixEpoch, DateTimeKind.Utc); + + public RunSummaryEntryDocument? LastRun { get; set; } + = null; + + public List Recent { get; set; } = new(); + + public RunSummaryCountersDocument Counters { get; set; } = new(); +} + +internal sealed class RunSummaryEntryDocument +{ + public string RunId { get; set; } = string.Empty; + + [BsonRepresentation(BsonType.String)] + public RunTrigger Trigger { get; set; } = RunTrigger.Cron; + + [BsonRepresentation(BsonType.String)] + public RunState State { get; set; } = RunState.Planning; + + public DateTime CreatedAt { get; set; } = DateTime.SpecifyKind(DateTime.UnixEpoch, DateTimeKind.Utc); + + public DateTime? StartedAt { get; set; } + = null; + + public DateTime? FinishedAt { get; set; } + = null; + + public RunStats Stats { get; set; } = RunStats.Empty; + + [BsonIgnoreIfNull] + public string? Error { get; set; } + = null; +} + +internal sealed class RunSummaryCountersDocument +{ + public int Total { get; set; } + = 0; + + public int Planning { get; set; } + = 0; + + public int Queued { get; set; } + = 0; + + public int Running { get; set; } + = 0; + + public int Completed { get; set; } + = 0; + + public int Error { get; set; } + = 0; + + public int Cancelled { get; set; } + = 0; + + public int TotalDeltas { get; set; } + = 0; + + public int TotalNewCriticals { get; set; } + = 0; + + public int TotalNewHigh { get; set; } + = 0; + + public int TotalNewMedium { get; set; } + = 0; + + public int TotalNewLow { get; set; } + = 0; +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs index fa74d09f..28ada0d8 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs @@ -17,15 +17,17 @@ internal sealed class EnsureSchedulerCollectionsMigration : ISchedulerMongoMigra { ArgumentNullException.ThrowIfNull(context); - var requiredCollections = new[] - { - context.Options.SchedulesCollection, - context.Options.RunsCollection, - context.Options.ImpactSnapshotsCollection, - context.Options.AuditCollection, - context.Options.LocksCollection, - context.Options.MigrationsCollection - }; + var requiredCollections = new[] + { + context.Options.SchedulesCollection, + context.Options.RunsCollection, + context.Options.ImpactSnapshotsCollection, + context.Options.AuditCollection, + context.Options.RunSummariesCollection, + context.Options.GraphJobsCollection, + context.Options.LocksCollection, + context.Options.MigrationsCollection + }; var cursor = await context.Database .ListCollectionNamesAsync(cancellationToken: cancellationToken) diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs index 22698899..4ab94c6a 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs @@ -13,12 +13,14 @@ internal sealed class EnsureSchedulerIndexesMigration : ISchedulerMongoMigration { ArgumentNullException.ThrowIfNull(context); - await EnsureSchedulesIndexesAsync(context, cancellationToken).ConfigureAwait(false); - await EnsureRunsIndexesAsync(context, cancellationToken).ConfigureAwait(false); - await EnsureImpactSnapshotsIndexesAsync(context, cancellationToken).ConfigureAwait(false); - await EnsureAuditIndexesAsync(context, cancellationToken).ConfigureAwait(false); - await EnsureLocksIndexesAsync(context, cancellationToken).ConfigureAwait(false); - } + await EnsureSchedulesIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureRunsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureImpactSnapshotsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureAuditIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureRunSummariesIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureGraphJobsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureLocksIndexesAsync(context, cancellationToken).ConfigureAwait(false); + } private static async Task EnsureSchedulesIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) { @@ -120,11 +122,11 @@ internal sealed class EnsureSchedulerIndexesMigration : ISchedulerMongoMigration .ConfigureAwait(false); } - private static async Task EnsureAuditIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) - { - var collection = context.Database.GetCollection(context.Options.AuditCollection); - - var tenantOccurred = new CreateIndexModel( + private static async Task EnsureAuditIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.AuditCollection); + + var tenantOccurred = new CreateIndexModel( Builders.IndexKeys .Ascending("tenantId") .Descending("occurredAt"), @@ -142,9 +144,72 @@ internal sealed class EnsureSchedulerIndexesMigration : ISchedulerMongoMigration PartialFilterExpression = Builders.Filter.Exists("correlationId", true) }); - await collection.Indexes.CreateManyAsync(new[] { tenantOccurred, correlationIndex }, cancellationToken: cancellationToken) - .ConfigureAwait(false); - } + await collection.Indexes.CreateManyAsync(new[] { tenantOccurred, correlationIndex }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + private static async Task EnsureRunSummariesIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.RunSummariesCollection); + + var tenantSchedule = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Ascending("scheduleId"), + new CreateIndexOptions + { + Name = "tenant_schedule", + Unique = true + }); + + var tenantUpdated = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Descending("updatedAt"), + new CreateIndexOptions + { + Name = "tenant_updatedAt_desc" + }); + + await collection.Indexes.CreateManyAsync(new[] { tenantSchedule, tenantUpdated }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + private static async Task EnsureGraphJobsIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.GraphJobsCollection); + + var tenantKindCreated = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Ascending("kind") + .Descending("createdAt"), + new CreateIndexOptions + { + Name = "tenant_kind_createdAt_desc" + }); + + var tenantStatus = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Ascending("status"), + new CreateIndexOptions + { + Name = "tenant_status" + }); + + var snapshotIndex = new CreateIndexModel( + Builders.IndexKeys + .Ascending("graphSnapshotId"), + new CreateIndexOptions + { + Name = "graphSnapshot_lookup", + PartialFilterExpression = Builders.Filter.Exists("graphSnapshotId", true) + }); + + await collection.Indexes.CreateManyAsync(new[] { tenantKindCreated, tenantStatus, snapshotIndex }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } private static async Task EnsureLocksIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) { diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs index 72a600b6..231b7d5f 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs +++ b/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs @@ -13,11 +13,15 @@ public sealed class SchedulerMongoOptions public string SchedulesCollection { get; set; } = "schedules"; - public string RunsCollection { get; set; } = "runs"; - - public string ImpactSnapshotsCollection { get; set; } = "impact_snapshots"; - - public string AuditCollection { get; set; } = "audit"; + public string RunsCollection { get; set; } = "runs"; + + public string ImpactSnapshotsCollection { get; set; } = "impact_snapshots"; + + public string AuditCollection { get; set; } = "audit"; + + public string RunSummariesCollection { get; set; } = "run_summaries"; + + public string GraphJobsCollection { get; set; } = "graph_jobs"; public string LocksCollection { get; set; } = "locks"; diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Projections/RunSummaryProjection.cs b/src/StellaOps.Scheduler.Storage.Mongo/Projections/RunSummaryProjection.cs new file mode 100644 index 00000000..f71a1317 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Projections/RunSummaryProjection.cs @@ -0,0 +1,36 @@ +using System.Collections.Immutable; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Projections; + +public sealed record RunSummaryProjection( + string TenantId, + string ScheduleId, + DateTimeOffset UpdatedAt, + RunSummarySnapshot? LastRun, + ImmutableArray Recent, + RunSummaryCounters Counters); + +public sealed record RunSummarySnapshot( + string RunId, + RunTrigger Trigger, + RunState State, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? FinishedAt, + RunStats Stats, + string? Error); + +public sealed record RunSummaryCounters( + int Total, + int Planning, + int Queued, + int Running, + int Completed, + int Error, + int Cancelled, + int TotalDeltas, + int TotalNewCriticals, + int TotalNewHigh, + int TotalNewMedium, + int TotalNewLow); diff --git a/src/StellaOps.Scheduler.Storage.Mongo/README.md b/src/StellaOps.Scheduler.Storage.Mongo/README.md index 241e4298..7a314bf8 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/README.md +++ b/src/StellaOps.Scheduler.Storage.Mongo/README.md @@ -22,4 +22,18 @@ All timestamps are persisted as UTC (`+00:00`). Empty selector filters remain em 4. Follow `docs/11_DATA_SCHEMAS.md` for index requirements; update that doc if storage diverges. 5. Register `AddSchedulerMongoStorage` in the host and call `ISchedulerMongoInitializer.EnsureMigrationsAsync` during bootstrap so collections/indexes are created before workers/web APIs start. -With these artefacts in place the dependency on SCHED-MODELS-16-101/102 is cleared—storage work can move to DOING. +With these artefacts in place the dependency on SCHED-MODELS-16-101/102 is cleared—storage work can move to DOING. + +## Repositories & Sessions (Sprint 16) + +- `AddSchedulerMongoStorage` now registers the scheduler repositories: +- `IScheduleRepository` — CRUD helpers with tenant scoping and soft-delete markers (`deletedAt`, `deletedBy`). +- `IRunRepository` — create/update/list helpers sorted by `createdAt`, honouring the TTL index on `finishedAt`. +- `IImpactSnapshotRepository` — stores canonical `ImpactSet` snapshots with deterministic selector digests. +- `IAuditRepository` — append/list audit log entries with optional category/schedule/run filters. +- `IRunSummaryRepository` — persists the `run_summaries` materialised view keyed by tenant/schedule. +- `IRunSummaryService` — projects run updates into the materialised view (latest run + rolling counters for the last 20 updates). +- `ISchedulerAuditService` — convenience layer that stamps IDs/timestamps and writes `AuditRecord` instances with consistent metadata. +- `ISchedulerMongoSessionFactory` provides causal-consistent Mongo sessions (default `CausalConsistency = true`) that repositories accept for read-after-write flows. +- All repositories persist canonical JSON via `CanonicalJsonSerializer`; helper mappers strip internal fields before materialising DTOs. +- Soft-deleted schedules keep historical documents but are excluded from queries unless `ScheduleQueryOptions.IncludeDeleted` is set; deletions also disable the schedule and bump `updatedAt/updatedBy`. diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/AuditQueryOptions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/AuditQueryOptions.cs new file mode 100644 index 00000000..67a22fc2 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/AuditQueryOptions.cs @@ -0,0 +1,32 @@ +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +/// +/// Filters applied when querying scheduler audit records. +/// +public sealed class AuditQueryOptions +{ + /// + /// Optional audit category filter (e.g., "scheduler"). + /// + public string? Category { get; init; } + + /// + /// Optional schedule identifier filter. + /// + public string? ScheduleId { get; init; } + + /// + /// Optional run identifier filter. + /// + public string? RunId { get; init; } + + /// + /// Lower bound for audit occurrence timestamp. + /// + public DateTimeOffset? Since { get; init; } + + /// + /// Maximum number of records to return. + /// + public int? Limit { get; init; } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/AuditRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/AuditRepository.cs new file mode 100644 index 00000000..68927f7c --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/AuditRepository.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Serialization; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal sealed class AuditRepository : IAuditRepository +{ + private static readonly FilterDefinitionBuilder Filter = Builders.Filter; + private static readonly SortDefinitionBuilder Sort = Builders.Sort; + + private readonly IMongoCollection _collection; + + public AuditRepository(SchedulerMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.AuditCollection); + } + + public async Task InsertAsync( + AuditRecord record, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(record); + + var document = AuditRecordDocumentMapper.ToBsonDocument(record); + if (session is null) + { + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.InsertOneAsync(session, document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + public async Task> ListAsync( + string tenantId, + AuditQueryOptions? options = null, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + options ??= new AuditQueryOptions(); + + var filters = new List> + { + Filter.Eq("tenantId", tenantId) + }; + + if (!string.IsNullOrWhiteSpace(options.Category)) + { + filters.Add(Filter.Eq("category", options.Category)); + } + + if (!string.IsNullOrWhiteSpace(options.ScheduleId)) + { + filters.Add(Filter.Eq("scheduleId", options.ScheduleId)); + } + + if (!string.IsNullOrWhiteSpace(options.RunId)) + { + filters.Add(Filter.Eq("runId", options.RunId)); + } + + if (options.Since is { } since) + { + filters.Add(Filter.Gte("occurredAt", since.ToUniversalTime().ToString("O"))); + } + + var combined = Filter.And(filters); + var find = session is null + ? _collection.Find(combined) + : _collection.Find(session, combined); + + var limit = options.Limit is { } specified && specified > 0 ? specified : 100; + var documents = await find + .Sort(Sort.Descending("occurredAt")) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents.Select(AuditRecordDocumentMapper.FromBsonDocument).ToArray(); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/GraphJobRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/GraphJobRepository.cs new file mode 100644 index 00000000..bf8cf4d7 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/GraphJobRepository.cs @@ -0,0 +1,126 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Serialization; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal sealed class GraphJobRepository : IGraphJobRepository +{ + private static readonly FilterDefinitionBuilder Filter = Builders.Filter; + private readonly IMongoCollection _collection; + + public GraphJobRepository(SchedulerMongoContext context) + { + ArgumentNullException.ThrowIfNull(context); + _collection = context.Database.GetCollection(context.Options.GraphJobsCollection); + } + + public async Task InsertAsync(GraphBuildJob job, CancellationToken cancellationToken = default) + { + var document = GraphJobDocumentMapper.ToBsonDocument(job); + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken = default) + { + var document = GraphJobDocumentMapper.ToBsonDocument(job); + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default) + { + var filter = BuildIdFilter(tenantId, jobId, "build"); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : GraphJobDocumentMapper.ToGraphBuildJob(document); + } + + public async Task GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default) + { + var filter = BuildIdFilter(tenantId, jobId, "overlay"); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : GraphJobDocumentMapper.ToGraphOverlayJob(document); + } + + public async Task ReplaceAsync(GraphBuildJob job, CancellationToken cancellationToken = default) + { + var document = GraphJobDocumentMapper.ToBsonDocument(job); + var filter = BuildIdFilter(job.TenantId, job.Id, "build"); + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = false }, cancellationToken).ConfigureAwait(false); + return job; + } + + public async Task ReplaceAsync(GraphOverlayJob job, CancellationToken cancellationToken = default) + { + var document = GraphJobDocumentMapper.ToBsonDocument(job); + var filter = BuildIdFilter(job.TenantId, job.Id, "overlay"); + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = false }, cancellationToken).ConfigureAwait(false); + return job; + } + + public async Task> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default) + { + var filters = new List> + { + Filter.Eq("tenantId", tenantId), + Filter.Eq("kind", "build") + }; + + if (status is { } s) + { + filters.Add(Filter.Eq("status", s.ToString().ToLowerInvariant())); + } + + var filter = Filter.And(filters); + var cursor = await _collection.Find(filter) + .Sort(Builders.Sort.Descending("createdAt")) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(GraphJobDocumentMapper.ToGraphBuildJob).ToArray(); + } + + public async Task> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default) + { + var filters = new List> + { + Filter.Eq("tenantId", tenantId), + Filter.Eq("kind", "overlay") + }; + + if (status is { } s) + { + filters.Add(Filter.Eq("status", s.ToString().ToLowerInvariant())); + } + + var filter = Filter.And(filters); + var cursor = await _collection.Find(filter) + .Sort(Builders.Sort.Descending("createdAt")) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(GraphJobDocumentMapper.ToGraphOverlayJob).ToArray(); + } + + public async Task> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken = default) + { + var filter = Filter.And( + Filter.Eq("tenantId", tenantId), + Filter.Eq("kind", "overlay")); + + var cursor = await _collection.Find(filter) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(GraphJobDocumentMapper.ToGraphOverlayJob).ToArray(); + } + + private static FilterDefinition BuildIdFilter(string tenantId, string jobId, string kind) + => Filter.And( + Filter.Eq("_id", jobId), + Filter.Eq("tenantId", tenantId), + Filter.Eq("kind", kind)); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IAuditRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IAuditRepository.cs new file mode 100644 index 00000000..847e5f40 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IAuditRepository.cs @@ -0,0 +1,18 @@ +using MongoDB.Driver; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +public interface IAuditRepository +{ + Task InsertAsync( + AuditRecord record, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task> ListAsync( + string tenantId, + AuditQueryOptions? options = null, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IGraphJobRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IGraphJobRepository.cs new file mode 100644 index 00000000..58f66456 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IGraphJobRepository.cs @@ -0,0 +1,24 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +public interface IGraphJobRepository +{ + Task InsertAsync(GraphBuildJob job, CancellationToken cancellationToken = default); + + Task InsertAsync(GraphOverlayJob job, CancellationToken cancellationToken = default); + + Task GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default); + + Task GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken = default); + + Task ReplaceAsync(GraphBuildJob job, CancellationToken cancellationToken = default); + + Task ReplaceAsync(GraphOverlayJob job, CancellationToken cancellationToken = default); + + Task> ListBuildJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default); + + Task> ListOverlayJobsAsync(string tenantId, GraphJobStatus? status, int limit, CancellationToken cancellationToken = default); + + Task> ListOverlayJobsAsync(string tenantId, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IImpactSnapshotRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IImpactSnapshotRepository.cs new file mode 100644 index 00000000..831b9c00 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IImpactSnapshotRepository.cs @@ -0,0 +1,22 @@ +using MongoDB.Driver; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +public interface IImpactSnapshotRepository +{ + Task UpsertAsync( + ImpactSet snapshot, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task GetBySnapshotIdAsync( + string snapshotId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task GetLatestBySelectorAsync( + Selector selector, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IRunRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IRunRepository.cs new file mode 100644 index 00000000..c07c81bb --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IRunRepository.cs @@ -0,0 +1,29 @@ +using MongoDB.Driver; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +public interface IRunRepository +{ + Task InsertAsync( + Run run, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task UpdateAsync( + Run run, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task GetAsync( + string tenantId, + string runId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task> ListAsync( + string tenantId, + RunQueryOptions? options = null, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IRunSummaryRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IRunSummaryRepository.cs new file mode 100644 index 00000000..8c3f291e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IRunSummaryRepository.cs @@ -0,0 +1,19 @@ +using StellaOps.Scheduler.Storage.Mongo.Documents; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal interface IRunSummaryRepository +{ + Task GetAsync( + string tenantId, + string scheduleId, + CancellationToken cancellationToken = default); + + Task> ListAsync( + string tenantId, + CancellationToken cancellationToken = default); + + Task UpsertAsync( + RunSummaryDocument document, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IScheduleRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IScheduleRepository.cs new file mode 100644 index 00000000..80d358ef --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/IScheduleRepository.cs @@ -0,0 +1,32 @@ +using MongoDB.Driver; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +public interface IScheduleRepository +{ + Task UpsertAsync( + Schedule schedule, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task GetAsync( + string tenantId, + string scheduleId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task> ListAsync( + string tenantId, + ScheduleQueryOptions? options = null, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); + + Task SoftDeleteAsync( + string tenantId, + string scheduleId, + string deletedBy, + DateTimeOffset deletedAt, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ImpactSnapshotRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ImpactSnapshotRepository.cs new file mode 100644 index 00000000..80181961 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ImpactSnapshotRepository.cs @@ -0,0 +1,94 @@ +using System; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Serialization; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal sealed class ImpactSnapshotRepository : IImpactSnapshotRepository +{ + private static readonly FilterDefinitionBuilder Filter = Builders.Filter; + private static readonly SortDefinitionBuilder Sort = Builders.Sort; + + private readonly IMongoCollection _collection; + + public ImpactSnapshotRepository(SchedulerMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.ImpactSnapshotsCollection); + } + + public async Task UpsertAsync( + ImpactSet snapshot, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(snapshot); + + var document = ImpactSetDocumentMapper.ToBsonDocument(snapshot); + var filter = Filter.Eq("_id", document["_id"]); + var options = new ReplaceOptions { IsUpsert = true }; + + if (session is null) + { + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + } + + public async Task GetBySnapshotIdAsync( + string snapshotId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(snapshotId)) + { + throw new ArgumentException("Snapshot id must be provided.", nameof(snapshotId)); + } + + var filter = Filter.Eq("_id", snapshotId); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : ImpactSetDocumentMapper.FromBsonDocument(document); + } + + public async Task GetLatestBySelectorAsync( + Selector selector, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(selector); + if (string.IsNullOrWhiteSpace(selector.TenantId)) + { + throw new ArgumentException("Selector tenantId is required to resolve impact snapshots.", nameof(selector)); + } + + var digest = ImpactSetDocumentMapper.ComputeSelectorDigest(selector); + var filters = Filter.And( + Filter.Eq("selectorDigest", digest), + Filter.Eq("selector.tenantId", selector.TenantId)); + + var find = session is null + ? _collection.Find(filters) + : _collection.Find(session, filters); + + var document = await find + .Sort(Sort.Descending("generatedAt")) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + return document is null ? null : ImpactSetDocumentMapper.FromBsonDocument(document); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunQueryOptions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunQueryOptions.cs new file mode 100644 index 00000000..6ffe0ca1 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunQueryOptions.cs @@ -0,0 +1,35 @@ +using System.Collections.Immutable; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +/// +/// Filters applied when listing scheduler runs. +/// +public sealed class RunQueryOptions +{ + /// + /// Optional schedule identifier to scope the list. + /// + public string? ScheduleId { get; init; } + + /// + /// Optional set of run states to include. When empty all states are returned. + /// + public ImmutableArray States { get; init; } = ImmutableArray.Empty; + + /// + /// Optional lower bound for creation timestamp (UTC). + /// + public DateTimeOffset? CreatedAfter { get; init; } + + /// + /// Maximum number of runs to return (default 50 when unspecified). + /// + public int? Limit { get; init; } + + /// + /// Sort order flag. Defaults to descending by createdAt. + /// + public bool SortAscending { get; init; } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunRepository.cs new file mode 100644 index 00000000..8b7c4be5 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunRepository.cs @@ -0,0 +1,153 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Serialization; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal sealed class RunRepository : IRunRepository +{ + private const int DefaultListLimit = 50; + + private static readonly FilterDefinitionBuilder Filter = Builders.Filter; + private static readonly SortDefinitionBuilder Sort = Builders.Sort; + + private readonly IMongoCollection _collection; + + public RunRepository(SchedulerMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.RunsCollection); + } + + public async Task InsertAsync( + Run run, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(run); + + var document = RunDocumentMapper.ToBsonDocument(run); + if (session is null) + { + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.InsertOneAsync(session, document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + public async Task UpdateAsync( + Run run, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(run); + + var document = RunDocumentMapper.ToBsonDocument(run); + var filter = Filter.And( + Filter.Eq("_id", run.Id), + Filter.Eq("tenantId", run.TenantId)); + + var options = new ReplaceOptions { IsUpsert = false }; + ReplaceOneResult result; + if (session is null) + { + result = await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + result = await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + + return result.MatchedCount > 0; + } + + public async Task GetAsync( + string tenantId, + string runId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + if (string.IsNullOrWhiteSpace(runId)) + { + throw new ArgumentException("Run id must be provided.", nameof(runId)); + } + + var filter = Filter.And( + Filter.Eq("_id", runId), + Filter.Eq("tenantId", tenantId)); + + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : RunDocumentMapper.FromBsonDocument(document); + } + + public async Task> ListAsync( + string tenantId, + RunQueryOptions? options = null, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + options ??= new RunQueryOptions(); + var filters = new List> + { + Filter.Eq("tenantId", tenantId) + }; + + if (!string.IsNullOrWhiteSpace(options.ScheduleId)) + { + filters.Add(Filter.Eq("scheduleId", options.ScheduleId)); + } + + if (options.States.Length > 0) + { + filters.Add(Filter.In("state", options.States.Select(state => state.ToString().ToLowerInvariant()))); + } + + if (options.CreatedAfter is { } createdAfter) + { + filters.Add(Filter.Gt("createdAt", createdAfter.ToUniversalTime().UtcDateTime)); + } + + var combined = Filter.And(filters); + + var find = session is null + ? _collection.Find(combined) + : _collection.Find(session, combined); + + var limit = options.Limit is { } specified && specified > 0 ? specified : DefaultListLimit; + find = find.Limit(limit); + + var sortDefinition = options.SortAscending + ? Sort.Ascending("createdAt") + : Sort.Descending("createdAt"); + + find = find.Sort(sortDefinition); + + var documents = await find.ToListAsync(cancellationToken).ConfigureAwait(false); + return documents.Select(RunDocumentMapper.FromBsonDocument).ToArray(); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunSummaryRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunSummaryRepository.cs new file mode 100644 index 00000000..324de7b1 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/RunSummaryRepository.cs @@ -0,0 +1,79 @@ +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Documents; +using StellaOps.Scheduler.Storage.Mongo.Internal; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal sealed class RunSummaryRepository : IRunSummaryRepository +{ + private readonly IMongoCollection _collection; + + public RunSummaryRepository(SchedulerMongoContext context) + { + ArgumentNullException.ThrowIfNull(context); + _collection = context.Database.GetCollection(context.Options.RunSummariesCollection); + } + + public async Task GetAsync( + string tenantId, + string scheduleId, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + if (string.IsNullOrWhiteSpace(scheduleId)) + { + throw new ArgumentException("Schedule id must be provided.", nameof(scheduleId)); + } + + var filter = Builders.Filter.Eq(document => document.Id, CreateDocumentId(tenantId, scheduleId)); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document; + } + + public async Task> ListAsync( + string tenantId, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + var filter = Builders.Filter.Eq(document => document.TenantId, tenantId); + var sort = Builders.Sort.Descending(document => document.UpdatedAt); + var documents = await _collection.Find(filter).Sort(sort).ToListAsync(cancellationToken).ConfigureAwait(false); + return documents; + } + + public Task UpsertAsync( + RunSummaryDocument document, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + if (string.IsNullOrWhiteSpace(document.TenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(document.TenantId)); + } + + if (string.IsNullOrWhiteSpace(document.ScheduleId)) + { + throw new ArgumentException("Schedule id must be provided.", nameof(document.ScheduleId)); + } + + document.Id = CreateDocumentId(document.TenantId, document.ScheduleId); + var filter = Builders.Filter.Eq(x => x.Id, document.Id); + return _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken); + } + + private static string CreateDocumentId(string tenantId, string scheduleId) + => string.Create(tenantId.Length + scheduleId.Length + 1, (tenantId, scheduleId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.scheduleId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ScheduleQueryOptions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ScheduleQueryOptions.cs new file mode 100644 index 00000000..d4f31aa8 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ScheduleQueryOptions.cs @@ -0,0 +1,22 @@ +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +/// +/// Filters applied when listing scheduler schedules. +/// +public sealed class ScheduleQueryOptions +{ + /// + /// When true, returns disabled schedules; otherwise disabled entries are excluded. + /// + public bool IncludeDisabled { get; init; } + + /// + /// When true, includes soft-deleted schedules; by default deleted entries are excluded. + /// + public bool IncludeDeleted { get; init; } + + /// + /// Optional maximum number of schedules to return. + /// + public int? Limit { get; init; } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ScheduleRepository.cs b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ScheduleRepository.cs new file mode 100644 index 00000000..af3b043a --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Repositories/ScheduleRepository.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Serialization; + +namespace StellaOps.Scheduler.Storage.Mongo.Repositories; + +internal sealed class ScheduleRepository : IScheduleRepository +{ + private static readonly FilterDefinitionBuilder Filter = Builders.Filter; + private static readonly UpdateDefinitionBuilder Update = Builders.Update; + + private readonly IMongoCollection _collection; + + public ScheduleRepository(SchedulerMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.SchedulesCollection); + } + + public async Task UpsertAsync( + Schedule schedule, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schedule); + + var document = ScheduleDocumentMapper.ToBsonDocument(schedule); + document.Remove("deletedAt"); + document.Remove("deletedBy"); + + var filter = Filter.And( + Filter.Eq("_id", schedule.Id), + Filter.Eq("tenantId", schedule.TenantId)); + + var options = new ReplaceOptions { IsUpsert = true }; + + if (session is null) + { + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + } + + public async Task GetAsync( + string tenantId, + string scheduleId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + if (string.IsNullOrWhiteSpace(scheduleId)) + { + throw new ArgumentException("Schedule id must be provided.", nameof(scheduleId)); + } + + var filter = Filter.And( + Filter.Eq("_id", scheduleId), + Filter.Eq("tenantId", tenantId), + Filter.Or( + Filter.Exists("deletedAt", false), + Filter.Eq("deletedAt", BsonNull.Value))); + + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : ScheduleDocumentMapper.FromBsonDocument(document); + } + + public async Task> ListAsync( + string tenantId, + ScheduleQueryOptions? options = null, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + options ??= new ScheduleQueryOptions(); + + var filters = new List> + { + Filter.Eq("tenantId", tenantId) + }; + + if (!options.IncludeDeleted) + { + filters.Add(Filter.Or( + Filter.Exists("deletedAt", false), + Filter.Eq("deletedAt", BsonNull.Value))); + } + + if (!options.IncludeDisabled) + { + filters.Add(Filter.Eq("enabled", true)); + } + + var combined = Filter.And(filters); + + var find = session is null + ? _collection.Find(combined) + : _collection.Find(session, combined); + + if (options.Limit is { } limit && limit > 0) + { + find = find.Limit(limit); + } + + var documents = await find.Sort(Builders.Sort.Ascending("name")) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents.Select(ScheduleDocumentMapper.FromBsonDocument).ToArray(); + } + + public async Task SoftDeleteAsync( + string tenantId, + string scheduleId, + string deletedBy, + DateTimeOffset deletedAt, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + if (string.IsNullOrWhiteSpace(scheduleId)) + { + throw new ArgumentException("Schedule id must be provided.", nameof(scheduleId)); + } + if (string.IsNullOrWhiteSpace(deletedBy)) + { + throw new ArgumentException("Deleted by must be provided.", nameof(deletedBy)); + } + + var filter = Filter.And( + Filter.Eq("_id", scheduleId), + Filter.Eq("tenantId", tenantId)); + + var utc = deletedAt.ToUniversalTime(); + var update = Update + .Set("deletedAt", utc.UtcDateTime) + .Set("deletedBy", deletedBy) + .Set("enabled", false) + .Set("updatedAt", utc.UtcDateTime) + .Set("updatedBy", deletedBy); + + UpdateResult result; + if (session is null) + { + result = await _collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + result = await _collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + return result.ModifiedCount > 0; + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Serialization/AuditRecordDocumentMapper.cs b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/AuditRecordDocumentMapper.cs new file mode 100644 index 00000000..d7b30db9 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/AuditRecordDocumentMapper.cs @@ -0,0 +1,23 @@ +using MongoDB.Bson; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Serialization; + +internal static class AuditRecordDocumentMapper +{ + public static BsonDocument ToBsonDocument(AuditRecord record) + { + ArgumentNullException.ThrowIfNull(record); + var json = CanonicalJsonSerializer.Serialize(record); + var document = BsonDocument.Parse(json); + document["_id"] = record.Id; + return document; + } + + public static AuditRecord FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var node = document.ToCanonicalJsonNode(); + return CanonicalJsonSerializer.Deserialize(node.ToCanonicalJson()); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Serialization/BsonDocumentJsonExtensions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/BsonDocumentJsonExtensions.cs new file mode 100644 index 00000000..6fe8970a --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/BsonDocumentJsonExtensions.cs @@ -0,0 +1,144 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; +using MongoDB.Bson; +using MongoDB.Bson.IO; + +namespace StellaOps.Scheduler.Storage.Mongo.Serialization; + +internal static class BsonDocumentJsonExtensions +{ + public static JsonNode ToCanonicalJsonNode(this BsonDocument document, params string[] fieldsToRemove) + { + ArgumentNullException.ThrowIfNull(document); + + var clone = document.DeepClone().AsBsonDocument; + clone.Remove("_id"); + + if (fieldsToRemove is { Length: > 0 }) + { + foreach (var field in fieldsToRemove) + { + clone.Remove(field); + } + } + + var json = clone.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + Indent = false, + }); + + var node = JsonNode.Parse(json) ?? throw new InvalidOperationException("Unable to parse BSON document JSON."); + return NormalizeExtendedJson(node); + } + + private static JsonNode NormalizeExtendedJson(JsonNode node) + { + if (node is JsonObject obj) + { + if (TryConvertExtendedDate(obj, out var replacement)) + { + return replacement; + } + + foreach (var property in obj.ToList()) + { + if (property.Value is null) + { + continue; + } + + var normalized = NormalizeExtendedJson(property.Value); + if (!ReferenceEquals(normalized, property.Value)) + { + obj[property.Key] = normalized; + } + } + + return obj; + } + + if (node is JsonArray array) + { + for (var i = 0; i < array.Count; i++) + { + if (array[i] is null) + { + continue; + } + + var normalized = NormalizeExtendedJson(array[i]!); + if (!ReferenceEquals(normalized, array[i])) + { + array[i] = normalized; + } + } + + return array; + } + + return node; + } + + private static bool TryConvertExtendedDate(JsonObject obj, out JsonNode replacement) + { + replacement = obj; + if (obj.Count != 1 || !obj.TryGetPropertyValue("$date", out var value) || value is null) + { + return false; + } + + if (value is JsonValue directValue) + { + if (directValue.TryGetValue(out string? dateString) && TryParseIso(dateString, out var iso)) + { + replacement = JsonValue.Create(iso); + return true; + } + + if (directValue.TryGetValue(out long epochMilliseconds)) + { + replacement = JsonValue.Create(DateTimeOffset.FromUnixTimeMilliseconds(epochMilliseconds).ToString("O")); + return true; + } + } + else if (value is JsonObject nested && + nested.TryGetPropertyValue("$numberLong", out var numberNode) && + numberNode is JsonValue numberValue && + numberValue.TryGetValue(out string? numberString) && + long.TryParse(numberString, NumberStyles.Integer, CultureInfo.InvariantCulture, out var ms)) + { + replacement = JsonValue.Create(DateTimeOffset.FromUnixTimeMilliseconds(ms).ToString("O")); + return true; + } + + return false; + } + + private static bool TryParseIso(string? value, out string iso) + { + iso = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var parsed)) + { + iso = parsed.ToUniversalTime().ToString("O"); + return true; + } + + return false; + } + + public static string ToCanonicalJson(this JsonNode node) + { + ArgumentNullException.ThrowIfNull(node); + return node.ToJsonString(new JsonSerializerOptions + { + WriteIndented = false + }); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Serialization/GraphJobDocumentMapper.cs b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/GraphJobDocumentMapper.cs new file mode 100644 index 00000000..965d56d3 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/GraphJobDocumentMapper.cs @@ -0,0 +1,125 @@ +using MongoDB.Bson; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Serialization; + +internal static class GraphJobDocumentMapper +{ + private const string PayloadField = "payload"; + + public static BsonDocument ToBsonDocument(GraphBuildJob job) + { + ArgumentNullException.ThrowIfNull(job); + + var payloadJson = CanonicalJsonSerializer.Serialize(job); + var payloadDocument = BsonDocument.Parse(payloadJson); + + var document = new BsonDocument + { + ["_id"] = job.Id, + ["tenantId"] = job.TenantId, + ["kind"] = "build", + ["status"] = job.Status.ToString().ToLowerInvariant(), + ["createdAt"] = job.CreatedAt.UtcDateTime, + ["attempts"] = job.Attempts, + [PayloadField] = payloadDocument + }; + + if (!string.IsNullOrWhiteSpace(job.GraphSnapshotId)) + { + document["graphSnapshotId"] = job.GraphSnapshotId; + } + + if (!string.IsNullOrWhiteSpace(job.CorrelationId)) + { + document["correlationId"] = job.CorrelationId; + } + + if (job.StartedAt is { } startedAt) + { + document["startedAt"] = startedAt.UtcDateTime; + } + + if (job.CompletedAt is { } completedAt) + { + document["completedAt"] = completedAt.UtcDateTime; + } + + if (!string.IsNullOrWhiteSpace(job.Error)) + { + document["error"] = job.Error; + } + + return document; + } + + public static BsonDocument ToBsonDocument(GraphOverlayJob job) + { + ArgumentNullException.ThrowIfNull(job); + + var payloadJson = CanonicalJsonSerializer.Serialize(job); + var payloadDocument = BsonDocument.Parse(payloadJson); + + var document = new BsonDocument + { + ["_id"] = job.Id, + ["tenantId"] = job.TenantId, + ["kind"] = "overlay", + ["status"] = job.Status.ToString().ToLowerInvariant(), + ["createdAt"] = job.CreatedAt.UtcDateTime, + ["attempts"] = job.Attempts, + [PayloadField] = payloadDocument + }; + + document["graphSnapshotId"] = job.GraphSnapshotId; + document["overlayKind"] = job.OverlayKind.ToString().ToLowerInvariant(); + document["overlayKey"] = job.OverlayKey; + + if (!string.IsNullOrWhiteSpace(job.BuildJobId)) + { + document["buildJobId"] = job.BuildJobId; + } + + if (!string.IsNullOrWhiteSpace(job.CorrelationId)) + { + document["correlationId"] = job.CorrelationId; + } + + if (job.StartedAt is { } startedAt) + { + document["startedAt"] = startedAt.UtcDateTime; + } + + if (job.CompletedAt is { } completedAt) + { + document["completedAt"] = completedAt.UtcDateTime; + } + + if (!string.IsNullOrWhiteSpace(job.Error)) + { + document["error"] = job.Error; + } + + return document; + } + + public static GraphBuildJob ToGraphBuildJob(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + var payloadDocument = document[PayloadField].AsBsonDocument; + var json = payloadDocument.ToJson(); + var job = CanonicalJsonSerializer.Deserialize(json); + return job; + } + + public static GraphOverlayJob ToGraphOverlayJob(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + var payloadDocument = document[PayloadField].AsBsonDocument; + var json = payloadDocument.ToJson(); + var job = CanonicalJsonSerializer.Deserialize(json); + return job; + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Serialization/ImpactSetDocumentMapper.cs b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/ImpactSetDocumentMapper.cs new file mode 100644 index 00000000..3586bfc2 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/ImpactSetDocumentMapper.cs @@ -0,0 +1,57 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using MongoDB.Bson; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Serialization; + +internal static class ImpactSetDocumentMapper +{ + private const string SelectorHashPrefix = "selector::"; + + public static BsonDocument ToBsonDocument(ImpactSet impactSet) + { + ArgumentNullException.ThrowIfNull(impactSet); + + var json = CanonicalJsonSerializer.Serialize(impactSet); + var document = BsonDocument.Parse(json); + document["_id"] = ComputeDocumentId(impactSet); + document["selectorDigest"] = ComputeSelectorDigest(impactSet); + return document; + } + + public static ImpactSet FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var node = document.ToCanonicalJsonNode(); + return CanonicalJsonSerializer.Deserialize(node.ToCanonicalJson()); + } + + private static string ComputeDocumentId(ImpactSet impactSet) + { + if (!string.IsNullOrWhiteSpace(impactSet.SnapshotId)) + { + return impactSet.SnapshotId!; + } + + var selectorJson = CanonicalJsonSerializer.Serialize(impactSet.Selector); + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(selectorJson)); + return SelectorHashPrefix + Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string ComputeSelectorDigest(ImpactSet impactSet) + { + return ComputeSelectorDigest(impactSet.Selector); + } + + public static string ComputeSelectorDigest(Selector selector) + { + ArgumentNullException.ThrowIfNull(selector); + var selectorJson = CanonicalJsonSerializer.Serialize(selector); + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(selectorJson)); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Serialization/RunDocumentMapper.cs b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/RunDocumentMapper.cs new file mode 100644 index 00000000..393d2524 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/RunDocumentMapper.cs @@ -0,0 +1,23 @@ +using MongoDB.Bson; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Serialization; + +internal static class RunDocumentMapper +{ + public static BsonDocument ToBsonDocument(Run run) + { + ArgumentNullException.ThrowIfNull(run); + var json = CanonicalJsonSerializer.Serialize(run); + var document = BsonDocument.Parse(json); + document["_id"] = run.Id; + return document; + } + + public static Run FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var node = document.ToCanonicalJsonNode(); + return CanonicalJsonSerializer.Deserialize(node.ToCanonicalJson()); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Serialization/ScheduleDocumentMapper.cs b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/ScheduleDocumentMapper.cs new file mode 100644 index 00000000..35f98313 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Serialization/ScheduleDocumentMapper.cs @@ -0,0 +1,25 @@ +using MongoDB.Bson; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Serialization; + +internal static class ScheduleDocumentMapper +{ + private static readonly string[] IgnoredFields = { "deletedAt", "deletedBy" }; + + public static BsonDocument ToBsonDocument(Schedule schedule) + { + ArgumentNullException.ThrowIfNull(schedule); + var json = CanonicalJsonSerializer.Serialize(schedule); + var document = BsonDocument.Parse(json); + document["_id"] = schedule.Id; + return document; + } + + public static Schedule FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var node = document.ToCanonicalJsonNode(IgnoredFields); + return CanonicalJsonSerializer.Deserialize(node.ToCanonicalJson()); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs index 859da90e..70f78e36 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs @@ -1,8 +1,12 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Scheduler.Storage.Mongo.Internal; -using StellaOps.Scheduler.Storage.Mongo.Migrations; -using StellaOps.Scheduler.Storage.Mongo.Options; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Migrations; +using StellaOps.Scheduler.Storage.Mongo.Options; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; +using StellaOps.Scheduler.Storage.Mongo.Sessions; namespace StellaOps.Scheduler.Storage.Mongo; @@ -16,11 +20,21 @@ public static class ServiceCollectionExtensions services.Configure(configuration); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddHostedService(); - - return services; - } -} + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Services/IRunSummaryService.cs b/src/StellaOps.Scheduler.Storage.Mongo/Services/IRunSummaryService.cs new file mode 100644 index 00000000..de10d16e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Services/IRunSummaryService.cs @@ -0,0 +1,20 @@ +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Projections; + +namespace StellaOps.Scheduler.Storage.Mongo.Services; + +public interface IRunSummaryService +{ + Task ProjectAsync( + Run run, + CancellationToken cancellationToken = default); + + Task GetAsync( + string tenantId, + string scheduleId, + CancellationToken cancellationToken = default); + + Task> ListAsync( + string tenantId, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Services/ISchedulerAuditService.cs b/src/StellaOps.Scheduler.Storage.Mongo/Services/ISchedulerAuditService.cs new file mode 100644 index 00000000..1943108b --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Services/ISchedulerAuditService.cs @@ -0,0 +1,10 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Services; + +public interface ISchedulerAuditService +{ + Task WriteAsync( + SchedulerAuditEvent auditEvent, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Services/RunSummaryService.cs b/src/StellaOps.Scheduler.Storage.Mongo/Services/RunSummaryService.cs new file mode 100644 index 00000000..a14dab43 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Services/RunSummaryService.cs @@ -0,0 +1,204 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Documents; +using StellaOps.Scheduler.Storage.Mongo.Projections; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.Storage.Mongo.Services; + +internal sealed class RunSummaryService : IRunSummaryService +{ + private const int RecentLimit = 20; + + private readonly IRunSummaryRepository _repository; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public RunSummaryService( + IRunSummaryRepository repository, + TimeProvider? timeProvider, + ILogger logger) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ProjectAsync( + Run run, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(run); + if (string.IsNullOrWhiteSpace(run.ScheduleId)) + { + throw new ArgumentException("Run must contain a scheduleId to project summary data.", nameof(run)); + } + + var document = await _repository + .GetAsync(run.TenantId, run.ScheduleId!, cancellationToken) + .ConfigureAwait(false) + ?? new RunSummaryDocument + { + TenantId = run.TenantId, + ScheduleId = run.ScheduleId!, + }; + + UpdateDocument(document, run); + document.UpdatedAt = _timeProvider.GetUtcNow().UtcDateTime; + + await _repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false); + _logger.LogDebug( + "Projected run summary for tenant {TenantId} schedule {ScheduleId} using run {RunId}.", + run.TenantId, + run.ScheduleId, + run.Id); + return ToProjection(document); + } + + public async Task GetAsync( + string tenantId, + string scheduleId, + CancellationToken cancellationToken = default) + { + var document = await _repository + .GetAsync(tenantId, scheduleId, cancellationToken) + .ConfigureAwait(false); + + return document is null ? null : ToProjection(document); + } + + public async Task> ListAsync( + string tenantId, + CancellationToken cancellationToken = default) + { + var documents = await _repository.ListAsync(tenantId, cancellationToken).ConfigureAwait(false); + return documents.Select(ToProjection).ToArray(); + } + + private static void UpdateDocument(RunSummaryDocument document, Run run) + { + var entry = document.Recent.FirstOrDefault(item => string.Equals(item.RunId, run.Id, StringComparison.Ordinal)); + if (entry is null) + { + entry = new RunSummaryEntryDocument + { + RunId = run.Id, + }; + document.Recent.Add(entry); + } + + entry.Trigger = run.Trigger; + entry.State = run.State; + entry.CreatedAt = run.CreatedAt.UtcDateTime; + entry.StartedAt = run.StartedAt?.UtcDateTime; + entry.FinishedAt = run.FinishedAt?.UtcDateTime; + entry.Error = run.Error; + entry.Stats = run.Stats; + + document.Recent = document.Recent + .OrderByDescending(item => item.CreatedAt) + .ThenByDescending(item => item.RunId, StringComparer.Ordinal) + .Take(RecentLimit) + .ToList(); + + document.LastRun = document.Recent.FirstOrDefault(); + document.Counters = ComputeCounters(document.Recent); + } + + private static RunSummaryCountersDocument ComputeCounters(IEnumerable entries) + { + var counters = new RunSummaryCountersDocument(); + + foreach (var entry in entries) + { + counters.Total++; + switch (entry.State) + { + case RunState.Planning: + counters.Planning++; + break; + case RunState.Queued: + counters.Queued++; + break; + case RunState.Running: + counters.Running++; + break; + case RunState.Completed: + counters.Completed++; + break; + case RunState.Error: + counters.Error++; + break; + case RunState.Cancelled: + counters.Cancelled++; + break; + default: + break; + } + + counters.TotalDeltas += entry.Stats.Deltas; + counters.TotalNewCriticals += entry.Stats.NewCriticals; + counters.TotalNewHigh += entry.Stats.NewHigh; + counters.TotalNewMedium += entry.Stats.NewMedium; + counters.TotalNewLow += entry.Stats.NewLow; + } + + return counters; + } + + private static RunSummaryProjection ToProjection(RunSummaryDocument document) + { + var updatedAt = new DateTimeOffset(DateTime.SpecifyKind(document.UpdatedAt, DateTimeKind.Utc)); + var lastRun = document.LastRun is null + ? null + : ToSnapshot(document.LastRun); + + var recent = document.Recent + .Select(ToSnapshot) + .ToImmutableArray(); + + var counters = new RunSummaryCounters( + document.Counters.Total, + document.Counters.Planning, + document.Counters.Queued, + document.Counters.Running, + document.Counters.Completed, + document.Counters.Error, + document.Counters.Cancelled, + document.Counters.TotalDeltas, + document.Counters.TotalNewCriticals, + document.Counters.TotalNewHigh, + document.Counters.TotalNewMedium, + document.Counters.TotalNewLow); + + return new RunSummaryProjection( + document.TenantId, + document.ScheduleId, + updatedAt, + lastRun, + recent, + counters); + } + + private static RunSummarySnapshot ToSnapshot(RunSummaryEntryDocument entry) + { + var createdAt = new DateTimeOffset(DateTime.SpecifyKind(entry.CreatedAt, DateTimeKind.Utc)); + DateTimeOffset? startedAt = entry.StartedAt is null + ? null + : new DateTimeOffset(DateTime.SpecifyKind(entry.StartedAt.Value, DateTimeKind.Utc)); + DateTimeOffset? finishedAt = entry.FinishedAt is null + ? null + : new DateTimeOffset(DateTime.SpecifyKind(entry.FinishedAt.Value, DateTimeKind.Utc)); + + return new RunSummarySnapshot( + entry.RunId, + entry.Trigger, + entry.State, + createdAt, + startedAt, + finishedAt, + entry.Stats, + entry.Error); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Services/SchedulerAuditEvent.cs b/src/StellaOps.Scheduler.Storage.Mongo/Services/SchedulerAuditEvent.cs new file mode 100644 index 00000000..145dda56 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Services/SchedulerAuditEvent.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Storage.Mongo.Services; + +public sealed record SchedulerAuditEvent( + string TenantId, + string Category, + string Action, + AuditActor Actor, + string? EntityId = null, + string? ScheduleId = null, + string? RunId = null, + string? CorrelationId = null, + IReadOnlyDictionary? Metadata = null, + string? Message = null, + DateTimeOffset? OccurredAt = null, + string? AuditId = null); diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Services/SchedulerAuditService.cs b/src/StellaOps.Scheduler.Storage.Mongo/Services/SchedulerAuditService.cs new file mode 100644 index 00000000..8497ba97 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Services/SchedulerAuditService.cs @@ -0,0 +1,62 @@ +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.Storage.Mongo.Services; + +internal sealed class SchedulerAuditService : ISchedulerAuditService +{ + private readonly IAuditRepository _repository; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public SchedulerAuditService( + IAuditRepository repository, + TimeProvider? timeProvider, + ILogger logger) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task WriteAsync( + SchedulerAuditEvent auditEvent, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(auditEvent); + + var occurredAt = auditEvent.OccurredAt ?? _timeProvider.GetUtcNow(); + var identifier = string.IsNullOrWhiteSpace(auditEvent.AuditId) + ? $"audit_{Guid.NewGuid():N}" + : auditEvent.AuditId!.Trim(); + + var metadata = auditEvent.Metadata is null + ? Enumerable.Empty>() + : auditEvent.Metadata.Where(pair => !string.IsNullOrWhiteSpace(pair.Key) && !string.IsNullOrWhiteSpace(pair.Value)); + + var record = new AuditRecord( + identifier, + auditEvent.TenantId, + auditEvent.Category, + auditEvent.Action, + occurredAt, + auditEvent.Actor, + auditEvent.EntityId, + auditEvent.ScheduleId, + auditEvent.RunId, + auditEvent.CorrelationId, + metadata, + auditEvent.Message); + + await _repository.InsertAsync(record, session: null, cancellationToken).ConfigureAwait(false); + + _logger.LogDebug( + "Scheduler audit record persisted with id {AuditId} for tenant {TenantId} action {Action}.", + record.Id, + record.TenantId, + record.Action); + + return record; + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Sessions/ISchedulerMongoSessionFactory.cs b/src/StellaOps.Scheduler.Storage.Mongo/Sessions/ISchedulerMongoSessionFactory.cs new file mode 100644 index 00000000..809d5197 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Sessions/ISchedulerMongoSessionFactory.cs @@ -0,0 +1,18 @@ +using MongoDB.Driver; + +namespace StellaOps.Scheduler.Storage.Mongo.Sessions; + +/// +/// Provides helper methods for creating MongoDB sessions used by scheduler storage. +/// +public interface ISchedulerMongoSessionFactory +{ + /// + /// Starts a new client session applying the provided options. + /// + /// Session options (optional). + /// Cancellation token. + Task StartSessionAsync( + SchedulerMongoSessionOptions? options = null, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Sessions/SchedulerMongoSessionFactory.cs b/src/StellaOps.Scheduler.Storage.Mongo/Sessions/SchedulerMongoSessionFactory.cs new file mode 100644 index 00000000..ade8c7b1 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Sessions/SchedulerMongoSessionFactory.cs @@ -0,0 +1,32 @@ +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Internal; + +namespace StellaOps.Scheduler.Storage.Mongo.Sessions; + +internal sealed class SchedulerMongoSessionFactory : ISchedulerMongoSessionFactory +{ + private readonly SchedulerMongoContext _context; + + public SchedulerMongoSessionFactory(SchedulerMongoContext context) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + } + + public Task StartSessionAsync( + SchedulerMongoSessionOptions? options = null, + CancellationToken cancellationToken = default) + { + var clientOptions = new ClientSessionOptions + { + CausalConsistency = options?.CausalConsistency ?? true, + }; + + if (options?.ReadPreference is not null) + { + clientOptions.DefaultTransactionOptions = new TransactionOptions( + readPreference: options.ReadPreference); + } + + return _context.Client.StartSessionAsync(clientOptions, cancellationToken); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Sessions/SchedulerMongoSessionOptions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Sessions/SchedulerMongoSessionOptions.cs new file mode 100644 index 00000000..c65a6c90 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Sessions/SchedulerMongoSessionOptions.cs @@ -0,0 +1,19 @@ +using MongoDB.Driver; + +namespace StellaOps.Scheduler.Storage.Mongo.Sessions; + +/// +/// Options controlling MongoDB client sessions for scheduler storage operations. +/// +public sealed class SchedulerMongoSessionOptions +{ + /// + /// When true (default), the session enables causal consistency for read operations following writes. + /// + public bool CausalConsistency { get; init; } = true; + + /// + /// Optional read preference override applied to the session when specified. + /// + public ReadPreference? ReadPreference { get; init; } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj b/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj index f3f01fcc..9445065c 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj +++ b/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj @@ -1,19 +1,19 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - - + + + net10.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md b/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md index 3ace7727..1a879b46 100644 --- a/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md +++ b/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md @@ -5,5 +5,5 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCHED-STORAGE-16-201 | DONE (2025-10-19) | Scheduler Storage Guild | SCHED-MODELS-16-101 | Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. | Migration scripts and indexes implemented; integration tests cover CRUD paths. | -| SCHED-STORAGE-16-202 | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. | Unit tests pass; TTL/soft delete validated; documentation updated. | -| SCHED-STORAGE-16-203 | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Audit/logging pipeline + run stats materialized views for UI. | Audit entries persisted; stats queries efficient; docs capture usage. | +| SCHED-STORAGE-16-202 | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. | Unit tests pass; TTL/soft delete validated; documentation updated. | +| SCHED-STORAGE-16-203 | DONE (2025-10-26) | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Audit/logging pipeline + run stats materialized views for UI. | Audit entries persisted; stats queries efficient; docs capture usage. | diff --git a/src/StellaOps.Scheduler.WebService.Tests/CartographerWebhookClientTests.cs b/src/StellaOps.Scheduler.WebService.Tests/CartographerWebhookClientTests.cs new file mode 100644 index 00000000..5711528e --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/CartographerWebhookClientTests.cs @@ -0,0 +1,140 @@ +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.WebService.GraphJobs; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class CartographerWebhookClientTests +{ + [Fact] + public async Task NotifyAsync_PostsPayload_WhenEnabled() + { + var handler = new RecordingHandler(); + var httpClient = new HttpClient(handler); + var options = Microsoft.Extensions.Options.Options.Create(new SchedulerCartographerOptions + { + Webhook = + { + Enabled = true, + Endpoint = "https://cartographer.local/hooks/graph-completed", + ApiKeyHeader = "X-Api-Key", + ApiKey = "secret" + } + }); + using var loggerFactory = LoggerFactory.Create(builder => builder.AddDebug()); + var client = new CartographerWebhookClient(httpClient, new OptionsMonitorStub(options), loggerFactory.CreateLogger()); + + var job = new GraphBuildJob( + id: "gbj_test", + tenantId: "tenant-alpha", + sbomId: "sbom", + sbomVersionId: "sbom_v1", + sbomDigest: "sha256:" + new string('a', 64), + status: GraphJobStatus.Completed, + trigger: GraphBuildJobTrigger.Backfill, + createdAt: DateTimeOffset.UtcNow, + graphSnapshotId: "snap", + attempts: 1, + cartographerJobId: "carto-123", + correlationId: "corr-1", + startedAt: null, + completedAt: DateTimeOffset.UtcNow, + error: null, + metadata: Array.Empty>()); + + var notification = new GraphJobCompletionNotification( + job.TenantId, + GraphJobQueryType.Build, + GraphJobStatus.Completed, + DateTimeOffset.UtcNow, + GraphJobResponse.From(job), + "oras://snap/result", + "corr-1", + null); + + await client.NotifyAsync(notification, CancellationToken.None); + + Assert.NotNull(handler.LastRequest); + Assert.Equal("https://cartographer.local/hooks/graph-completed", handler.LastRequest.RequestUri!.ToString()); + Assert.True(handler.LastRequest.Headers.TryGetValues("X-Api-Key", out var values) && values!.Single() == "secret"); + var json = JsonSerializer.Deserialize(handler.LastPayload!); + Assert.Equal("gbj_test", json.GetProperty("jobId").GetString()); + Assert.Equal("tenant-alpha", json.GetProperty("tenantId").GetString()); + } + + [Fact] + public async Task NotifyAsync_Skips_WhenDisabled() + { + var handler = new RecordingHandler(); + var httpClient = new HttpClient(handler); + var options = Microsoft.Extensions.Options.Options.Create(new SchedulerCartographerOptions()); + using var loggerFactory = LoggerFactory.Create(builder => builder.AddDebug()); + var client = new CartographerWebhookClient(httpClient, new OptionsMonitorStub(options), loggerFactory.CreateLogger()); + + var job = new GraphOverlayJob( + id: "goj-test", + tenantId: "tenant-alpha", + graphSnapshotId: "snap", + overlayKind: GraphOverlayKind.Policy, + overlayKey: "policy@1", + status: GraphJobStatus.Completed, + trigger: GraphOverlayJobTrigger.Manual, + createdAt: DateTimeOffset.UtcNow, + subjects: Array.Empty(), + attempts: 1, + correlationId: null, + startedAt: null, + completedAt: DateTimeOffset.UtcNow, + error: null, + metadata: Array.Empty>()); + + var notification = new GraphJobCompletionNotification( + job.TenantId, + GraphJobQueryType.Overlay, + GraphJobStatus.Completed, + DateTimeOffset.UtcNow, + GraphJobResponse.From(job), + null, + null, + null); + + await client.NotifyAsync(notification, CancellationToken.None); + + Assert.Null(handler.LastRequest); + } + + private sealed class RecordingHandler : HttpMessageHandler + { + public HttpRequestMessage? LastRequest { get; private set; } + public string? LastPayload { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + LastRequest = request; + LastPayload = request.Content is null ? null : request.Content.ReadAsStringAsync(cancellationToken).Result; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK)); + } + } + + private sealed class OptionsMonitorStub : IOptionsMonitor where T : class + { + private readonly IOptions _options; + + public OptionsMonitorStub(IOptions options) + { + _options = options; + } + + public T CurrentValue => _options.Value; + + public T Get(string? name) => _options.Value; + + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/EventWebhookEndpointTests.cs b/src/StellaOps.Scheduler.WebService.Tests/EventWebhookEndpointTests.cs new file mode 100644 index 00000000..e1ec21cc --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/EventWebhookEndpointTests.cs @@ -0,0 +1,128 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class EventWebhookEndpointTests : IClassFixture> +{ + static EventWebhookEndpointTests() + { + Environment.SetEnvironmentVariable("Scheduler__Events__Webhooks__Feedser__HmacSecret", FeedserSecret); + Environment.SetEnvironmentVariable("Scheduler__Events__Webhooks__Feedser__Enabled", "true"); + Environment.SetEnvironmentVariable("Scheduler__Events__Webhooks__Vexer__HmacSecret", VexerSecret); + Environment.SetEnvironmentVariable("Scheduler__Events__Webhooks__Vexer__Enabled", "true"); + } + + private const string FeedserSecret = "feedser-secret"; + private const string VexerSecret = "vexer-secret"; + + private readonly WebApplicationFactory _factory; + + public EventWebhookEndpointTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task FeedserWebhook_AcceptsValidSignature() + { + using var client = _factory.CreateClient(); + var payload = new + { + exportId = "feedser-exp-1", + changedProductKeys = new[] { "pkg:rpm/openssl", "pkg:deb/nginx" }, + kev = new[] { "CVE-2024-0001" }, + window = new { from = DateTimeOffset.UtcNow.AddHours(-1), to = DateTimeOffset.UtcNow } + }; + + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + using var request = new HttpRequestMessage(HttpMethod.Post, "/events/feedser-export") + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + request.Headers.TryAddWithoutValidation("X-Scheduler-Signature", ComputeSignature(FeedserSecret, json)); + + var response = await client.SendAsync(request); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + } + + [Fact] + public async Task FeedserWebhook_RejectsInvalidSignature() + { + using var client = _factory.CreateClient(); + var payload = new + { + exportId = "feedser-exp-2", + changedProductKeys = new[] { "pkg:nuget/log4net" } + }; + + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + using var request = new HttpRequestMessage(HttpMethod.Post, "/events/feedser-export") + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + request.Headers.TryAddWithoutValidation("X-Scheduler-Signature", "sha256=invalid"); + + var response = await client.SendAsync(request); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task VexerWebhook_HonoursRateLimit() + { + using var restrictedFactory = _factory.WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, configuration) => + { + configuration.AddInMemoryCollection(new Dictionary + { + ["Scheduler:Events:Webhooks:Vexer:RateLimitRequests"] = "1", + ["Scheduler:Events:Webhooks:Vexer:RateLimitWindowSeconds"] = "60" + }); + }); + }); + + using var client = restrictedFactory.CreateClient(); + var payload = new + { + exportId = "vexer-exp-1", + changedClaims = new[] + { + new { productKey = "pkg:deb/openssl", vulnerabilityId = "CVE-2024-1234", status = "affected" } + } + }; + + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + + using var first = new HttpRequestMessage(HttpMethod.Post, "/events/vexer-export") + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + first.Headers.TryAddWithoutValidation("X-Scheduler-Signature", ComputeSignature(VexerSecret, json)); + var firstResponse = await client.SendAsync(first); + Assert.Equal(HttpStatusCode.Accepted, firstResponse.StatusCode); + + using var second = new HttpRequestMessage(HttpMethod.Post, "/events/vexer-export") + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + second.Headers.TryAddWithoutValidation("X-Scheduler-Signature", ComputeSignature(VexerSecret, json)); + var secondResponse = await client.SendAsync(second); + Assert.Equal((HttpStatusCode)429, secondResponse.StatusCode); + Assert.True(secondResponse.Headers.Contains("Retry-After")); + } + + private static string ComputeSignature(string secret, string payload) + { + using var hmac = new HMACSHA256(Encoding.UTF8.GetBytes(secret)); + var hash = hmac.ComputeHash(Encoding.UTF8.GetBytes(payload)); + return "sha256=" + Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/GlobalUsings.cs b/src/StellaOps.Scheduler.WebService.Tests/GlobalUsings.cs new file mode 100644 index 00000000..fb6a9c39 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/GlobalUsings.cs @@ -0,0 +1,6 @@ +global using System.Net.Http.Json; +global using System.Text.Json; +global using System.Text.Json.Serialization; +global using System.Threading.Tasks; +global using Microsoft.AspNetCore.Mvc.Testing; +global using Xunit; diff --git a/src/StellaOps.Scheduler.WebService.Tests/GraphJobEndpointTests.cs b/src/StellaOps.Scheduler.WebService.Tests/GraphJobEndpointTests.cs new file mode 100644 index 00000000..4790f025 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/GraphJobEndpointTests.cs @@ -0,0 +1,110 @@ +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class GraphJobEndpointTests : IClassFixture +{ + private readonly SchedulerWebApplicationFactory _factory; + + public GraphJobEndpointTests(SchedulerWebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task CreateGraphBuildJob_RequiresGraphWriteScope() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-alpha"); + + var response = await client.PostAsJsonAsync("/graphs/build", new + { + sbomId = "sbom-test", + sbomVersionId = "sbom-ver", + sbomDigest = "sha256:" + new string('a', 64) + }); + + Assert.Equal(System.Net.HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task CreateGraphBuildJob_AndList() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-alpha"); + client.DefaultRequestHeaders.Add("X-Scopes", string.Join(' ', StellaOpsScopes.GraphWrite, StellaOpsScopes.GraphRead)); + + var createResponse = await client.PostAsJsonAsync("/graphs/build", new + { + sbomId = "sbom-alpha", + sbomVersionId = "sbom-alpha-v1", + sbomDigest = "sha256:" + new string('b', 64), + metadata = new { source = "test" } + }); + + createResponse.EnsureSuccessStatusCode(); + + var listResponse = await client.GetAsync("/graphs/jobs"); + listResponse.EnsureSuccessStatusCode(); + + var json = await listResponse.Content.ReadAsStringAsync(); + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + Assert.True(root.TryGetProperty("jobs", out var jobs)); + Assert.True(jobs.GetArrayLength() >= 1); + var first = jobs[0]; + Assert.Equal("build", first.GetProperty("kind").GetString()); + Assert.Equal("tenant-alpha", first.GetProperty("tenantId").GetString()); + Assert.Equal("pending", first.GetProperty("status").GetString()); + } + + [Fact] + public async Task CompleteOverlayJob_UpdatesStatusAndMetrics() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-bravo"); + client.DefaultRequestHeaders.Add("X-Scopes", string.Join(' ', StellaOpsScopes.GraphWrite, StellaOpsScopes.GraphRead)); + + var createOverlay = await client.PostAsJsonAsync("/graphs/overlays", new + { + graphSnapshotId = "graph_snap_20251026", + overlayKind = "policy", + overlayKey = "policy@2025-10-01", + subjects = new[] { "artifact/service-api" } + }); + + createOverlay.EnsureSuccessStatusCode(); + var createdJson = await createOverlay.Content.ReadAsStringAsync(); + using var createdDoc = JsonDocument.Parse(createdJson); + var jobId = createdDoc.RootElement.GetProperty("id").GetString(); + Assert.False(string.IsNullOrEmpty(jobId)); + + var completeResponse = await client.PostAsJsonAsync("/graphs/hooks/completed", new + { + jobId = jobId, + jobType = "Overlay", + status = "Completed", + occurredAt = DateTimeOffset.UtcNow, + correlationId = "corr-123", + resultUri = "oras://cartographer/snapshots/graph_snap_20251026" + }); + + completeResponse.EnsureSuccessStatusCode(); + var completedJson = await completeResponse.Content.ReadAsStringAsync(); + using var completedDoc = JsonDocument.Parse(completedJson); + Assert.Equal("completed", completedDoc.RootElement.GetProperty("status").GetString()); + + var metricsResponse = await client.GetAsync("/graphs/overlays/lag"); + metricsResponse.EnsureSuccessStatusCode(); + var metricsJson = await metricsResponse.Content.ReadAsStringAsync(); + using var metricsDoc = JsonDocument.Parse(metricsJson); + var metricsRoot = metricsDoc.RootElement; + Assert.Equal("tenant-bravo", metricsRoot.GetProperty("tenantId").GetString()); + Assert.True(metricsRoot.GetProperty("completed").GetInt32() >= 1); + var recent = metricsRoot.GetProperty("recentCompleted"); + Assert.True(recent.GetArrayLength() >= 1); + var entry = recent[0]; + Assert.Equal(jobId, entry.GetProperty("jobId").GetString()); + Assert.Equal("corr-123", entry.GetProperty("correlationId").GetString()); + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/GraphJobEventPublisherTests.cs b/src/StellaOps.Scheduler.WebService.Tests/GraphJobEventPublisherTests.cs new file mode 100644 index 00000000..60cc5a68 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/GraphJobEventPublisherTests.cs @@ -0,0 +1,151 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.WebService.GraphJobs; +using StellaOps.Scheduler.WebService.GraphJobs.Events; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class GraphJobEventPublisherTests +{ + [Fact] + public async Task PublishAsync_WritesEventJson_WhenEnabled() + { + var options = Microsoft.Extensions.Options.Options.Create(new SchedulerEventsOptions + { + GraphJobs = { Enabled = true } + }); + var loggerProvider = new ListLoggerProvider(); + using var loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(loggerProvider)); + var publisher = new GraphJobEventPublisher(new OptionsMonitorStub(options), loggerFactory.CreateLogger()); + + var buildJob = new GraphBuildJob( + id: "gbj_test", + tenantId: "tenant-alpha", + sbomId: "sbom", + sbomVersionId: "sbom_v1", + sbomDigest: "sha256:" + new string('a', 64), + status: GraphJobStatus.Completed, + trigger: GraphBuildJobTrigger.SbomVersion, + createdAt: DateTimeOffset.UtcNow, + graphSnapshotId: "graph_snap", + attempts: 1, + cartographerJobId: "carto", + correlationId: "corr", + startedAt: DateTimeOffset.UtcNow.AddSeconds(-30), + completedAt: DateTimeOffset.UtcNow, + error: null, + metadata: Array.Empty>()); + + var notification = new GraphJobCompletionNotification( + buildJob.TenantId, + GraphJobQueryType.Build, + GraphJobStatus.Completed, + DateTimeOffset.UtcNow, + GraphJobResponse.From(buildJob), + "oras://result", + "corr", + null); + + await publisher.PublishAsync(notification, CancellationToken.None); + + var message = Assert.Single(loggerProvider.Messages); + Assert.Contains("\"kind\":\"scheduler.graph.job.completed\"", message); + Assert.Contains("\"tenant\":\"tenant-alpha\"", message); + Assert.Contains("\"resultUri\":\"oras://result\"", message); + } + + [Fact] + public async Task PublishAsync_Suppressed_WhenDisabled() + { + var options = Microsoft.Extensions.Options.Options.Create(new SchedulerEventsOptions()); + var loggerProvider = new ListLoggerProvider(); + using var loggerFactory = LoggerFactory.Create(builder => builder.AddProvider(loggerProvider)); + var publisher = new GraphJobEventPublisher(new OptionsMonitorStub(options), loggerFactory.CreateLogger()); + + var overlayJob = new GraphOverlayJob( + id: "goj_test", + tenantId: "tenant-alpha", + graphSnapshotId: "graph_snap", + buildJobId: null, + overlayKind: GraphOverlayKind.Policy, + overlayKey: "policy@1", + subjects: Array.Empty(), + status: GraphJobStatus.Completed, + trigger: GraphOverlayJobTrigger.Policy, + createdAt: DateTimeOffset.UtcNow, + attempts: 1, + correlationId: null, + startedAt: DateTimeOffset.UtcNow.AddSeconds(-10), + completedAt: DateTimeOffset.UtcNow, + error: null, + metadata: Array.Empty>()); + + var notification = new GraphJobCompletionNotification( + overlayJob.TenantId, + GraphJobQueryType.Overlay, + GraphJobStatus.Completed, + DateTimeOffset.UtcNow, + GraphJobResponse.From(overlayJob), + null, + null, + null); + + await publisher.PublishAsync(notification, CancellationToken.None); + + Assert.DoesNotContain(loggerProvider.Messages, message => message.Contains(GraphJobEventKinds.GraphJobCompleted, StringComparison.Ordinal)); + } + +private sealed class OptionsMonitorStub : IOptionsMonitor where T : class + { + private readonly IOptions _options; + + public OptionsMonitorStub(IOptions options) + { + _options = options; + } + + public T CurrentValue => _options.Value; + + public T Get(string? name) => _options.Value; + + public IDisposable? OnChange(Action listener) => null; + } + + private sealed class ListLoggerProvider : ILoggerProvider + { + private readonly ListLogger _logger = new(); + + public IList Messages => _logger.Messages; + + public ILogger CreateLogger(string categoryName) => _logger; + + public void Dispose() + { + } + + private sealed class ListLogger : ILogger + { + public IList Messages { get; } = new List(); + + public IDisposable BeginScope(TState state) where TState : notnull => NullDisposable.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Messages.Add(formatter(state, exception)); + } + + private sealed class NullDisposable : IDisposable + { + public static readonly NullDisposable Instance = new(); + public void Dispose() + { + } + } + } + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs b/src/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs new file mode 100644 index 00000000..6059019a --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/RunEndpointTests.cs @@ -0,0 +1,104 @@ +using System.Linq; +using System.Text.Json; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class RunEndpointTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public RunEndpointTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task CreateListCancelRun() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-runs"); + client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview"); + + var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new + { + name = "RunSchedule", + cronExpression = "0 3 * * *", + timezone = "UTC", + mode = "analysis-only", + selection = new + { + scope = "all-images" + } + }); + + scheduleResponse.EnsureSuccessStatusCode(); + var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync(); + var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString(); + Assert.False(string.IsNullOrEmpty(scheduleId)); + + var createRun = await client.PostAsJsonAsync("/api/v1/scheduler/runs", new + { + scheduleId, + trigger = "manual" + }); + + createRun.EnsureSuccessStatusCode(); + Assert.Equal(System.Net.HttpStatusCode.Created, createRun.StatusCode); + var runJson = await createRun.Content.ReadFromJsonAsync(); + var runId = runJson.GetProperty("run").GetProperty("id").GetString(); + Assert.False(string.IsNullOrEmpty(runId)); + Assert.Equal("planning", runJson.GetProperty("run").GetProperty("state").GetString()); + + var listResponse = await client.GetAsync("/api/v1/scheduler/runs"); + listResponse.EnsureSuccessStatusCode(); + var listJson = await listResponse.Content.ReadFromJsonAsync(); + Assert.True(listJson.GetProperty("runs").EnumerateArray().Any()); + + var cancelResponse = await client.PostAsync($"/api/v1/scheduler/runs/{runId}/cancel", null); + cancelResponse.EnsureSuccessStatusCode(); + var cancelled = await cancelResponse.Content.ReadFromJsonAsync(); + Assert.Equal("cancelled", cancelled.GetProperty("run").GetProperty("state").GetString()); + + var getResponse = await client.GetAsync($"/api/v1/scheduler/runs/{runId}"); + getResponse.EnsureSuccessStatusCode(); + var runDetail = await getResponse.Content.ReadFromJsonAsync(); + Assert.Equal("cancelled", runDetail.GetProperty("run").GetProperty("state").GetString()); + } + + [Fact] + public async Task PreviewImpactForSchedule() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-preview"); + client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read scheduler.runs.write scheduler.runs.read scheduler.runs.preview"); + + var scheduleResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new + { + name = "PreviewSchedule", + cronExpression = "0 5 * * *", + timezone = "UTC", + mode = "analysis-only", + selection = new + { + scope = "all-images" + } + }); + + scheduleResponse.EnsureSuccessStatusCode(); + var scheduleJson = await scheduleResponse.Content.ReadFromJsonAsync(); + var scheduleId = scheduleJson.GetProperty("schedule").GetProperty("id").GetString(); + Assert.False(string.IsNullOrEmpty(scheduleId)); + + var previewResponse = await client.PostAsJsonAsync("/api/v1/scheduler/runs/preview", new + { + scheduleId, + usageOnly = true, + sampleSize = 3 + }); + + previewResponse.EnsureSuccessStatusCode(); + var preview = await previewResponse.Content.ReadFromJsonAsync(); + Assert.True(preview.GetProperty("total").GetInt32() >= 0); + Assert.True(preview.GetProperty("sample").GetArrayLength() <= 3); + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/ScheduleEndpointTests.cs b/src/StellaOps.Scheduler.WebService.Tests/ScheduleEndpointTests.cs new file mode 100644 index 00000000..ee0b67cc --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/ScheduleEndpointTests.cs @@ -0,0 +1,88 @@ +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class ScheduleEndpointTests : IClassFixture> +{ + private readonly WebApplicationFactory _factory; + + public ScheduleEndpointTests(WebApplicationFactory factory) + { + _factory = factory; + } + + [Fact] + public async Task CreateListAndRetrieveSchedule() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-schedules"); + client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read"); + + var createResponse = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new + { + name = "Nightly", + cronExpression = "0 2 * * *", + timezone = "UTC", + mode = "analysis-only", + selection = new + { + scope = "all-images" + }, + notify = new + { + onNewFindings = true, + minSeverity = "medium", + includeKev = true + } + }); + + createResponse.EnsureSuccessStatusCode(); + var created = await createResponse.Content.ReadFromJsonAsync(); + var scheduleId = created.GetProperty("schedule").GetProperty("id").GetString(); + Assert.False(string.IsNullOrEmpty(scheduleId)); + + var listResponse = await client.GetAsync("/api/v1/scheduler/schedules"); + listResponse.EnsureSuccessStatusCode(); + var listJson = await listResponse.Content.ReadFromJsonAsync(); + var schedules = listJson.GetProperty("schedules"); + Assert.True(schedules.GetArrayLength() >= 1); + + var getResponse = await client.GetAsync($"/api/v1/scheduler/schedules/{scheduleId}"); + getResponse.EnsureSuccessStatusCode(); + var scheduleJson = await getResponse.Content.ReadFromJsonAsync(); + Assert.Equal("Nightly", scheduleJson.GetProperty("schedule").GetProperty("name").GetString()); + } + + [Fact] + public async Task PauseAndResumeSchedule() + { + using var client = _factory.CreateClient(); + client.DefaultRequestHeaders.Add("X-Tenant-Id", "tenant-controls"); + client.DefaultRequestHeaders.Add("X-Scopes", "scheduler.schedules.write scheduler.schedules.read"); + + var create = await client.PostAsJsonAsync("/api/v1/scheduler/schedules", new + { + name = "PauseResume", + cronExpression = "*/5 * * * *", + timezone = "UTC", + mode = "analysis-only", + selection = new + { + scope = "all-images" + } + }); + + create.EnsureSuccessStatusCode(); + var created = await create.Content.ReadFromJsonAsync(); + var scheduleId = created.GetProperty("schedule").GetProperty("id").GetString(); + Assert.False(string.IsNullOrEmpty(scheduleId)); + + var pauseResponse = await client.PostAsync($"/api/v1/scheduler/schedules/{scheduleId}/pause", null); + pauseResponse.EnsureSuccessStatusCode(); + var paused = await pauseResponse.Content.ReadFromJsonAsync(); + Assert.False(paused.GetProperty("schedule").GetProperty("enabled").GetBoolean()); + + var resumeResponse = await client.PostAsync($"/api/v1/scheduler/schedules/{scheduleId}/resume", null); + resumeResponse.EnsureSuccessStatusCode(); + var resumed = await resumeResponse.Content.ReadFromJsonAsync(); + Assert.True(resumed.GetProperty("schedule").GetProperty("enabled").GetBoolean()); + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs b/src/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs new file mode 100644 index 00000000..ccaf06b2 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/SchedulerWebApplicationFactory.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.Tests; + +public sealed class SchedulerWebApplicationFactory : WebApplicationFactory +{ + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.ConfigureAppConfiguration((_, configuration) => + { + configuration.AddInMemoryCollection(new[] + { + new KeyValuePair("Scheduler:Authority:Enabled", "false"), + new KeyValuePair("Scheduler:Cartographer:Webhook:Enabled", "false"), + new KeyValuePair("Scheduler:Events:GraphJobs:Enabled", "false"), + new KeyValuePair("Scheduler:Events:Webhooks:Feedser:Enabled", "true"), + new KeyValuePair("Scheduler:Events:Webhooks:Feedser:HmacSecret", "feedser-secret"), + new KeyValuePair("Scheduler:Events:Webhooks:Feedser:RateLimitRequests", "20"), + new KeyValuePair("Scheduler:Events:Webhooks:Feedser:RateLimitWindowSeconds", "60"), + new KeyValuePair("Scheduler:Events:Webhooks:Vexer:Enabled", "true"), + new KeyValuePair("Scheduler:Events:Webhooks:Vexer:HmacSecret", "vexer-secret"), + new KeyValuePair("Scheduler:Events:Webhooks:Vexer:RateLimitRequests", "20"), + new KeyValuePair("Scheduler:Events:Webhooks:Vexer:RateLimitWindowSeconds", "60") + }); + }); + + builder.ConfigureServices(services => + { + services.Configure(options => + { + options.Webhooks ??= new SchedulerInboundWebhooksOptions(); + options.Webhooks.Feedser ??= SchedulerWebhookOptions.CreateDefault("feedser"); + options.Webhooks.Vexer ??= SchedulerWebhookOptions.CreateDefault("vexer"); + options.Webhooks.Feedser.HmacSecret = "feedser-secret"; + options.Webhooks.Feedser.Enabled = true; + options.Webhooks.Vexer.HmacSecret = "vexer-secret"; + options.Webhooks.Vexer.Enabled = true; + }); + }); + } +} diff --git a/src/StellaOps.Scheduler.WebService.Tests/StellaOps.Scheduler.WebService.Tests.csproj b/src/StellaOps.Scheduler.WebService.Tests/StellaOps.Scheduler.WebService.Tests.csproj new file mode 100644 index 00000000..f9818719 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService.Tests/StellaOps.Scheduler.WebService.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + false + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scheduler.WebService/Auth/AnonymousAuthenticationHandler.cs b/src/StellaOps.Scheduler.WebService/Auth/AnonymousAuthenticationHandler.cs new file mode 100644 index 00000000..3209ffc8 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/AnonymousAuthenticationHandler.cs @@ -0,0 +1,26 @@ +using System.Security.Claims; +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Scheduler.WebService.Auth; + +internal sealed class AnonymousAuthenticationHandler : AuthenticationHandler +{ + public AnonymousAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var identity = new ClaimsIdentity(Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } +} diff --git a/src/StellaOps.Scheduler.WebService/Auth/ClaimsTenantContextAccessor.cs b/src/StellaOps.Scheduler.WebService/Auth/ClaimsTenantContextAccessor.cs new file mode 100644 index 00000000..1b0d50ee --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/ClaimsTenantContextAccessor.cs @@ -0,0 +1,27 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Http; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Scheduler.WebService.Auth; + +internal sealed class ClaimsTenantContextAccessor : ITenantContextAccessor +{ + public TenantContext GetTenant(HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var principal = context.User ?? throw new UnauthorizedAccessException("Authentication required."); + if (principal.Identity?.IsAuthenticated != true) + { + throw new UnauthorizedAccessException("Authentication required."); + } + + var tenant = principal.FindFirstValue(StellaOpsClaimTypes.Tenant); + if (string.IsNullOrWhiteSpace(tenant)) + { + throw new InvalidOperationException("Authenticated principal is missing required tenant claim."); + } + + return new TenantContext(tenant.Trim()); + } +} diff --git a/src/StellaOps.Scheduler.WebService/Auth/HeaderScopeAuthorizer.cs b/src/StellaOps.Scheduler.WebService/Auth/HeaderScopeAuthorizer.cs new file mode 100644 index 00000000..819cd4a8 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/HeaderScopeAuthorizer.cs @@ -0,0 +1,31 @@ +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Scheduler.WebService.Auth; + +internal sealed class HeaderScopeAuthorizer : IScopeAuthorizer +{ + private const string ScopeHeader = "X-Scopes"; + + public void EnsureScope(HttpContext context, string requiredScope) + { + if (!context.Request.Headers.TryGetValue(ScopeHeader, out var values)) + { + throw new UnauthorizedAccessException($"Missing required header '{ScopeHeader}'."); + } + + var scopeBuffer = string.Join(' ', values.ToArray()); + if (string.IsNullOrWhiteSpace(scopeBuffer)) + { + throw new UnauthorizedAccessException($"Header '{ScopeHeader}' cannot be empty."); + } + + var scopes = scopeBuffer + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (!scopes.Contains(requiredScope)) + { + throw new InvalidOperationException($"Missing required scope '{requiredScope}'."); + } + } +} diff --git a/src/StellaOps.Scheduler.WebService/Auth/HeaderTenantContextAccessor.cs b/src/StellaOps.Scheduler.WebService/Auth/HeaderTenantContextAccessor.cs new file mode 100644 index 00000000..2b431322 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/HeaderTenantContextAccessor.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Scheduler.WebService.Auth; + +internal sealed class HeaderTenantContextAccessor : ITenantContextAccessor +{ + private const string TenantHeader = "X-Tenant-Id"; + + public TenantContext GetTenant(HttpContext context) + { + if (!context.Request.Headers.TryGetValue(TenantHeader, out var values)) + { + throw new UnauthorizedAccessException($"Missing required header '{TenantHeader}'."); + } + + var tenantId = values.ToString().Trim(); + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new UnauthorizedAccessException($"Header '{TenantHeader}' cannot be empty."); + } + + return new TenantContext(tenantId); + } +} diff --git a/src/StellaOps.Scheduler.WebService/Auth/IScopeAuthorizer.cs b/src/StellaOps.Scheduler.WebService/Auth/IScopeAuthorizer.cs new file mode 100644 index 00000000..bf4d7edd --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/IScopeAuthorizer.cs @@ -0,0 +1,8 @@ +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Scheduler.WebService.Auth; + +public interface IScopeAuthorizer +{ + void EnsureScope(HttpContext context, string requiredScope); +} diff --git a/src/StellaOps.Scheduler.WebService/Auth/ITenantContextAccessor.cs b/src/StellaOps.Scheduler.WebService/Auth/ITenantContextAccessor.cs new file mode 100644 index 00000000..180045cc --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/ITenantContextAccessor.cs @@ -0,0 +1,10 @@ +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Scheduler.WebService.Auth; + +public interface ITenantContextAccessor +{ + TenantContext GetTenant(HttpContext context); +} + +public sealed record TenantContext(string TenantId); diff --git a/src/StellaOps.Scheduler.WebService/Auth/TokenScopeAuthorizer.cs b/src/StellaOps.Scheduler.WebService/Auth/TokenScopeAuthorizer.cs new file mode 100644 index 00000000..3397cc7d --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Auth/TokenScopeAuthorizer.cs @@ -0,0 +1,61 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Http; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Scheduler.WebService.Auth; + +internal sealed class TokenScopeAuthorizer : IScopeAuthorizer +{ + public void EnsureScope(HttpContext context, string requiredScope) + { + ArgumentNullException.ThrowIfNull(context); + + if (string.IsNullOrWhiteSpace(requiredScope)) + { + return; + } + + var principal = context.User ?? throw new UnauthorizedAccessException("Authentication required."); + if (principal.Identity?.IsAuthenticated != true) + { + throw new UnauthorizedAccessException("Authentication required."); + } + + var normalizedRequired = StellaOpsScopes.Normalize(requiredScope) ?? requiredScope.Trim().ToLowerInvariant(); + if (!HasScope(principal, normalizedRequired)) + { + throw new InvalidOperationException($"Missing required scope '{normalizedRequired}'."); + } + } + + private static bool HasScope(ClaimsPrincipal principal, string requiredScope) + { + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem)) + { + if (string.Equals(claim.Value, requiredScope, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var part in parts) + { + var normalized = StellaOpsScopes.Normalize(part); + if (normalized is not null && string.Equals(normalized, requiredScope, StringComparison.Ordinal)) + { + return true; + } + } + } + + return false; + } +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/EventWebhookEndpointExtensions.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/EventWebhookEndpointExtensions.cs new file mode 100644 index 00000000..f4e256ea --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/EventWebhookEndpointExtensions.cs @@ -0,0 +1,173 @@ +using System.ComponentModel.DataAnnotations; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +public static class EventWebhookEndpointExtensions +{ + public static void MapSchedulerEventWebhookEndpoints(this IEndpointRouteBuilder builder) + { + var group = builder.MapGroup("/events"); + + group.MapPost("/feedser-export", HandleFeedserExportAsync); + group.MapPost("/vexer-export", HandleVexerExportAsync); + } + + private static async Task HandleFeedserExportAsync( + HttpContext httpContext, + [FromServices] IOptionsMonitor options, + [FromServices] IWebhookRequestAuthenticator authenticator, + [FromServices] IWebhookRateLimiter rateLimiter, + [FromServices] IInboundExportEventSink sink, + CancellationToken cancellationToken) + { + var webhookOptions = options.CurrentValue.Webhooks.Feedser; + if (!webhookOptions.Enabled) + { + return Results.StatusCode(StatusCodes.Status503ServiceUnavailable); + } + + var readResult = await ReadPayloadAsync(httpContext, cancellationToken).ConfigureAwait(false); + if (!readResult.Succeeded) + { + return readResult.ErrorResult!; + } + + if (!rateLimiter.TryAcquire("feedser", webhookOptions.RateLimitRequests, webhookOptions.GetRateLimitWindow(), out var retryAfter)) + { + var response = Results.StatusCode(StatusCodes.Status429TooManyRequests); + if (retryAfter > TimeSpan.Zero) + { + httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retryAfter.TotalSeconds)).ToString(); + } + + return response; + } + + var authResult = await authenticator.AuthenticateAsync(httpContext, readResult.RawBody, webhookOptions, cancellationToken).ConfigureAwait(false); + if (!authResult.Succeeded) + { + return authResult.ToResult(); + } + + try + { + await sink.HandleFeedserAsync(readResult.Payload!, cancellationToken).ConfigureAwait(false); + return Results.Accepted(value: new { status = "accepted" }); + } + catch (ValidationException ex) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task HandleVexerExportAsync( + HttpContext httpContext, + [FromServices] IOptionsMonitor options, + [FromServices] IWebhookRequestAuthenticator authenticator, + [FromServices] IWebhookRateLimiter rateLimiter, + [FromServices] IInboundExportEventSink sink, + CancellationToken cancellationToken) + { + var webhookOptions = options.CurrentValue.Webhooks.Vexer; + if (!webhookOptions.Enabled) + { + return Results.StatusCode(StatusCodes.Status503ServiceUnavailable); + } + + var readResult = await ReadPayloadAsync(httpContext, cancellationToken).ConfigureAwait(false); + if (!readResult.Succeeded) + { + return readResult.ErrorResult!; + } + + if (!rateLimiter.TryAcquire("vexer", webhookOptions.RateLimitRequests, webhookOptions.GetRateLimitWindow(), out var retryAfter)) + { + var response = Results.StatusCode(StatusCodes.Status429TooManyRequests); + if (retryAfter > TimeSpan.Zero) + { + httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retryAfter.TotalSeconds)).ToString(); + } + + return response; + } + + var authResult = await authenticator.AuthenticateAsync(httpContext, readResult.RawBody, webhookOptions, cancellationToken).ConfigureAwait(false); + if (!authResult.Succeeded) + { + return authResult.ToResult(); + } + + try + { + await sink.HandleVexerAsync(readResult.Payload!, cancellationToken).ConfigureAwait(false); + return Results.Accepted(value: new { status = "accepted" }); + } + catch (ValidationException ex) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task> ReadPayloadAsync(HttpContext context, CancellationToken cancellationToken) + { + context.Request.EnableBuffering(); + + await using var buffer = new MemoryStream(); + await context.Request.Body.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + var bodyBytes = buffer.ToArray(); + context.Request.Body.Position = 0; + + try + { + var payload = JsonSerializer.Deserialize(bodyBytes, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + if (payload is null) + { + return RequestPayload.Failed(Results.BadRequest(new { error = "Request payload cannot be empty." })); + } + + return RequestPayload.Success(payload, bodyBytes); + } + catch (JsonException ex) + { + return RequestPayload.Failed(Results.BadRequest(new { error = ex.Message })); + } + catch (ValidationException ex) + { + return RequestPayload.Failed(Results.BadRequest(new { error = ex.Message })); + } + } + + private readonly struct RequestPayload + { + private RequestPayload(T? payload, byte[] rawBody, IResult? error, bool succeeded) + { + Payload = payload; + RawBody = rawBody; + ErrorResult = error; + Succeeded = succeeded; + } + + public T? Payload { get; } + + public byte[] RawBody { get; } + + public IResult? ErrorResult { get; } + + public bool Succeeded { get; } + + public static RequestPayload Success(T payload, byte[] rawBody) + => new(payload, rawBody, null, true); + + public static RequestPayload Failed(IResult error) + => new(default, Array.Empty(), error, false); + } +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/IInboundExportEventSink.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/IInboundExportEventSink.cs new file mode 100644 index 00000000..66c9df60 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/IInboundExportEventSink.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +public interface IInboundExportEventSink +{ + Task HandleFeedserAsync(FeedserExportEventRequest request, CancellationToken cancellationToken); + + Task HandleVexerAsync(VexerExportEventRequest request, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/IWebhookRateLimiter.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/IWebhookRateLimiter.cs new file mode 100644 index 00000000..0213f36c --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/IWebhookRateLimiter.cs @@ -0,0 +1,8 @@ +using System; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +public interface IWebhookRateLimiter +{ + bool TryAcquire(string key, int limit, TimeSpan window, out TimeSpan retryAfter); +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/IWebhookRequestAuthenticator.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/IWebhookRequestAuthenticator.cs new file mode 100644 index 00000000..11ea5ee2 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/IWebhookRequestAuthenticator.cs @@ -0,0 +1,107 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +public interface IWebhookRequestAuthenticator +{ + Task AuthenticateAsync(HttpContext context, ReadOnlyMemory body, SchedulerWebhookOptions options, CancellationToken cancellationToken); +} + +internal sealed class WebhookRequestAuthenticator : IWebhookRequestAuthenticator +{ + private readonly ILogger _logger; + + public WebhookRequestAuthenticator( + ILogger logger) + { + _logger = logger; + } + + public async Task AuthenticateAsync(HttpContext context, ReadOnlyMemory body, SchedulerWebhookOptions options, CancellationToken cancellationToken) + { + if (!options.Enabled) + { + return WebhookAuthenticationResult.Success(); + } + + if (options.RequireClientCertificate) + { + var certificate = context.Connection.ClientCertificate ?? await context.Connection.GetClientCertificateAsync(cancellationToken).ConfigureAwait(false); + if (certificate is null) + { + _logger.LogWarning("Webhook {Name} rejected request without client certificate.", options.Name); + return WebhookAuthenticationResult.Fail(StatusCodes.Status401Unauthorized, "Client certificate required."); + } + } + + if (!string.IsNullOrWhiteSpace(options.HmacSecret)) + { + var headerName = string.IsNullOrWhiteSpace(options.SignatureHeader) ? "X-Scheduler-Signature" : options.SignatureHeader; + if (!context.Request.Headers.TryGetValue(headerName, out var signatureValues)) + { + _logger.LogWarning("Webhook {Name} rejected request missing signature header {Header}.", options.Name, headerName); + return WebhookAuthenticationResult.Fail(StatusCodes.Status401Unauthorized, "Missing signature header."); + } + + var providedSignature = signatureValues.ToString(); + if (string.IsNullOrWhiteSpace(providedSignature)) + { + return WebhookAuthenticationResult.Fail(StatusCodes.Status401Unauthorized, "Signature header is empty."); + } + + if (!VerifySignature(body.Span, options.HmacSecret!, providedSignature)) + { + _logger.LogWarning("Webhook {Name} rejected request with invalid signature.", options.Name); + return WebhookAuthenticationResult.Fail(StatusCodes.Status401Unauthorized, "Invalid signature."); + } + } + + return WebhookAuthenticationResult.Success(); + } + + private static bool VerifySignature(ReadOnlySpan payload, string secret, string providedSignature) + { + byte[] secretBytes; + try + { + secretBytes = Convert.FromBase64String(secret); + } + catch (FormatException) + { + try + { + secretBytes = Convert.FromHexString(secret); + } + catch (FormatException) + { + secretBytes = Encoding.UTF8.GetBytes(secret); + } + } + + using var hmac = new HMACSHA256(secretBytes); + var hash = hmac.ComputeHash(payload.ToArray()); + var computedSignature = "sha256=" + Convert.ToHexString(hash).ToLowerInvariant(); + + return CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(computedSignature), + Encoding.UTF8.GetBytes(providedSignature.Trim())); + } +} + +public readonly record struct WebhookAuthenticationResult(bool Succeeded, int StatusCode, string? Message) +{ + public static WebhookAuthenticationResult Success() => new(true, StatusCodes.Status200OK, null); + + public static WebhookAuthenticationResult Fail(int statusCode, string message) => new(false, statusCode, message); + + public IResult ToResult() + => Succeeded ? Results.Ok() : Results.Json(new { error = Message ?? "Unauthorized" }, statusCode: StatusCode); +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/InMemoryWebhookRateLimiter.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/InMemoryWebhookRateLimiter.cs new file mode 100644 index 00000000..4643ccbc --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/InMemoryWebhookRateLimiter.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Caching.Memory; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +internal sealed class InMemoryWebhookRateLimiter : IWebhookRateLimiter, IDisposable +{ + private readonly MemoryCache _cache = new(new MemoryCacheOptions()); + + private readonly object _mutex = new(); + + public bool TryAcquire(string key, int limit, TimeSpan window, out TimeSpan retryAfter) + { + if (limit <= 0) + { + retryAfter = TimeSpan.Zero; + return true; + } + + retryAfter = TimeSpan.Zero; + var now = DateTimeOffset.UtcNow; + + lock (_mutex) + { + if (!_cache.TryGetValue(key, out Queue? hits)) + { + hits = new Queue(); + _cache.Set(key, hits, new MemoryCacheEntryOptions + { + SlidingExpiration = window.Add(window) + }); + } + + hits ??= new Queue(); + + while (hits.Count > 0 && now - hits.Peek() > window) + { + hits.Dequeue(); + } + + if (hits.Count >= limit) + { + var oldest = hits.Peek(); + retryAfter = (oldest + window) - now; + if (retryAfter < TimeSpan.Zero) + { + retryAfter = TimeSpan.Zero; + } + + return false; + } + + hits.Enqueue(now); + return true; + } + } + + public void Dispose() + { + _cache.Dispose(); + } +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/LoggingExportEventSink.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/LoggingExportEventSink.cs new file mode 100644 index 00000000..2bc2994c --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/LoggingExportEventSink.cs @@ -0,0 +1,33 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +internal sealed class LoggingExportEventSink : IInboundExportEventSink +{ + private readonly ILogger _logger; + + public LoggingExportEventSink(ILogger logger) + { + _logger = logger; + } + + public Task HandleFeedserAsync(FeedserExportEventRequest request, CancellationToken cancellationToken) + { + _logger.LogInformation( + "Received Feedser export webhook {ExportId} with {ChangedProducts} product keys.", + request.ExportId, + request.ChangedProductKeys.Count); + return Task.CompletedTask; + } + + public Task HandleVexerAsync(VexerExportEventRequest request, CancellationToken cancellationToken) + { + _logger.LogInformation( + "Received Vexer export webhook {ExportId} with {ChangedClaims} claim changes.", + request.ExportId, + request.ChangedClaims.Count); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scheduler.WebService/EventWebhooks/WebhookPayloads.cs b/src/StellaOps.Scheduler.WebService/EventWebhooks/WebhookPayloads.cs new file mode 100644 index 00000000..a9627df4 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/EventWebhooks/WebhookPayloads.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Linq; + +namespace StellaOps.Scheduler.WebService.EventWebhooks; + +public sealed record FeedserExportEventRequest( + string ExportId, + IReadOnlyList ChangedProductKeys, + IReadOnlyList? Kev, + WebhookEventWindow? Window) +{ + public string ExportId { get; } = ExportId?.Trim() ?? throw new ArgumentNullException(nameof(ExportId)); + + public IReadOnlyList ChangedProductKeys { get; } = NormalizeList(ChangedProductKeys, nameof(ChangedProductKeys)); + + public IReadOnlyList Kev { get; } = NormalizeList(Kev, nameof(Kev), allowEmpty: true); + + public WebhookEventWindow? Window { get; } = Window; + + private static IReadOnlyList NormalizeList(IReadOnlyList? source, string propertyName, bool allowEmpty = false) + { + if (source is null) + { + if (allowEmpty) + { + return ImmutableArray.Empty; + } + + throw new ValidationException($"{propertyName} must be specified."); + } + + var cleaned = source + .Where(item => !string.IsNullOrWhiteSpace(item)) + .Select(item => item.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (!allowEmpty && cleaned.Length == 0) + { + throw new ValidationException($"{propertyName} must contain at least one value."); + } + + return cleaned; + } +} + +public sealed record VexerExportEventRequest( + string ExportId, + IReadOnlyList ChangedClaims, + WebhookEventWindow? Window) +{ + public string ExportId { get; } = ExportId?.Trim() ?? throw new ArgumentNullException(nameof(ExportId)); + + public IReadOnlyList ChangedClaims { get; } = NormalizeClaims(ChangedClaims); + + public WebhookEventWindow? Window { get; } = Window; + + private static IReadOnlyList NormalizeClaims(IReadOnlyList? claims) + { + if (claims is null || claims.Count == 0) + { + throw new ValidationException("changedClaims must contain at least one entry."); + } + + foreach (var claim in claims) + { + claim.Validate(); + } + + return claims; + } +} + +public sealed record VexerClaimChange( + string ProductKey, + string VulnerabilityId, + string Status) +{ + public string ProductKey { get; } = Normalize(ProductKey, nameof(ProductKey)); + + public string VulnerabilityId { get; } = Normalize(VulnerabilityId, nameof(VulnerabilityId)); + + public string Status { get; } = Normalize(Status, nameof(Status)); + + internal void Validate() + { + _ = ProductKey; + _ = VulnerabilityId; + _ = Status; + } + + private static string Normalize(string value, string propertyName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ValidationException($"{propertyName} must be provided."); + } + + return value.Trim(); + } +} + +public sealed record WebhookEventWindow(DateTimeOffset? From, DateTimeOffset? To); diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/CartographerWebhookClient.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/CartographerWebhookClient.cs new file mode 100644 index 00000000..69eeffb7 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/CartographerWebhookClient.cs @@ -0,0 +1,102 @@ +using System.Net.Http.Headers; +using System.Net.Mime; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +internal sealed class CartographerWebhookClient : ICartographerWebhookClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + private readonly HttpClient _httpClient; + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + + public CartographerWebhookClient( + HttpClient httpClient, + IOptionsMonitor options, + ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task NotifyAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken) + { + var snapshot = _options.CurrentValue; + var webhook = snapshot.Webhook; + + if (!webhook.Enabled) + { + _logger.LogDebug("Cartographer webhook disabled; skipping notification for job {JobId}.", notification.Job.Id); + return; + } + + if (string.IsNullOrWhiteSpace(webhook.Endpoint)) + { + _logger.LogWarning("Cartographer webhook endpoint not configured; unable to notify for job {JobId}.", notification.Job.Id); + return; + } + + Uri endpointUri; + try + { + endpointUri = new Uri(webhook.Endpoint, UriKind.Absolute); + } + catch (Exception ex) + { + _logger.LogError(ex, "Invalid Cartographer webhook endpoint '{Endpoint}'.", webhook.Endpoint); + return; + } + + var payload = new + { + tenantId = notification.TenantId, + jobId = notification.Job.Id, + jobType = notification.Job.Kind, + status = notification.Status.ToString().ToLowerInvariant(), + occurredAt = notification.OccurredAt, + resultUri = notification.ResultUri, + correlationId = notification.CorrelationId, + job = notification.Job, + error = notification.Error + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, endpointUri) + { + Content = new StringContent(JsonSerializer.Serialize(payload, SerializerOptions), Encoding.UTF8, MediaTypeNames.Application.Json) + }; + + if (!string.IsNullOrWhiteSpace(webhook.ApiKey) && !string.IsNullOrWhiteSpace(webhook.ApiKeyHeader)) + { + request.Headers.TryAddWithoutValidation(webhook.ApiKeyHeader!, webhook.ApiKey); + } + + try + { + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + _logger.LogWarning("Cartographer webhook responded {StatusCode} for job {JobId}: {Body}", (int)response.StatusCode, notification.Job.Id, body); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to invoke Cartographer webhook for job {JobId}.", notification.Job.Id); + } + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobCompletedEvent.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobCompletedEvent.cs new file mode 100644 index 00000000..f24f1e94 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobCompletedEvent.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs.Events; + +internal sealed record GraphJobCompletedEvent +{ + [JsonPropertyName("eventId")] + public required string EventId { get; init; } + + [JsonPropertyName("kind")] + public required string Kind { get; init; } + + [JsonPropertyName("tenant")] + public required string Tenant { get; init; } + + [JsonPropertyName("ts")] + public required DateTimeOffset Timestamp { get; init; } + + [JsonPropertyName("payload")] + public required GraphJobCompletedPayload Payload { get; init; } + + [JsonPropertyName("attributes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IReadOnlyDictionary? Attributes { get; init; } +} + +internal sealed record GraphJobCompletedPayload +{ + [JsonPropertyName("jobType")] + public required string JobType { get; init; } + + [JsonPropertyName("status")] + public required GraphJobStatus Status { get; init; } + + [JsonPropertyName("occurredAt")] + public required DateTimeOffset OccurredAt { get; init; } + + [JsonPropertyName("job")] + public required GraphJobResponse Job { get; init; } + + [JsonPropertyName("resultUri")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ResultUri { get; init; } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventFactory.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventFactory.cs new file mode 100644 index 00000000..5814b148 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventFactory.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs.Events; + +internal static class GraphJobEventFactory +{ + public static GraphJobCompletedEvent Create(GraphJobCompletionNotification notification) + { + var eventId = Guid.CreateVersion7().ToString("n"); + var attributes = new Dictionary(StringComparer.Ordinal); + + if (!string.IsNullOrWhiteSpace(notification.CorrelationId)) + { + attributes["correlationId"] = notification.CorrelationId!; + } + + if (!string.IsNullOrWhiteSpace(notification.Error)) + { + attributes["error"] = notification.Error!; + } + + var payload = new GraphJobCompletedPayload + { + JobType = notification.JobType.ToString().ToLowerInvariant(), + Status = notification.Status, + OccurredAt = notification.OccurredAt, + Job = notification.Job, + ResultUri = notification.ResultUri + }; + + return new GraphJobCompletedEvent + { + EventId = eventId, + Kind = GraphJobEventKinds.GraphJobCompleted, + Tenant = notification.TenantId, + Timestamp = notification.OccurredAt, + Payload = payload, + Attributes = attributes.Count == 0 ? null : attributes + }; + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventKinds.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventKinds.cs new file mode 100644 index 00000000..12b26478 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventKinds.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scheduler.WebService.GraphJobs.Events; + +internal static class GraphJobEventKinds +{ + public const string GraphJobCompleted = "scheduler.graph.job.completed"; +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventPublisher.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventPublisher.cs new file mode 100644 index 00000000..649ebca7 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/Events/GraphJobEventPublisher.cs @@ -0,0 +1,41 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scheduler.WebService.Options; + +namespace StellaOps.Scheduler.WebService.GraphJobs.Events; + +internal sealed class GraphJobEventPublisher : IGraphJobCompletionPublisher +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + + public GraphJobEventPublisher( + IOptionsMonitor options, + ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task PublishAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken) + { + var options = _options.CurrentValue; + if (!options.GraphJobs.Enabled) + { + _logger.LogDebug("Graph job events disabled; skipping emission for {JobId}.", notification.Job.Id); + return Task.CompletedTask; + } + + var envelope = GraphJobEventFactory.Create(notification); + var json = JsonSerializer.Serialize(envelope, SerializerOptions); + _logger.LogInformation("{EventJson}", json); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphBuildJobRequest.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphBuildJobRequest.cs new file mode 100644 index 00000000..8d8cdc52 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphBuildJobRequest.cs @@ -0,0 +1,26 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record GraphBuildJobRequest +{ + [Required] + public string SbomId { get; init; } = string.Empty; + + [Required] + public string SbomVersionId { get; init; } = string.Empty; + + [Required] + public string SbomDigest { get; init; } = string.Empty; + + public string? GraphSnapshotId { get; init; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphBuildJobTrigger? Trigger { get; init; } + + public string? CorrelationId { get; init; } + + public IDictionary? Metadata { get; init; } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobCompletionNotification.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobCompletionNotification.cs new file mode 100644 index 00000000..b4666a33 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobCompletionNotification.cs @@ -0,0 +1,13 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record GraphJobCompletionNotification( + string TenantId, + GraphJobQueryType JobType, + GraphJobStatus Status, + DateTimeOffset OccurredAt, + GraphJobResponse Job, + string? ResultUri, + string? CorrelationId, + string? Error); diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobCompletionRequest.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobCompletionRequest.cs new file mode 100644 index 00000000..800ebf37 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobCompletionRequest.cs @@ -0,0 +1,30 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record GraphJobCompletionRequest +{ + [Required] + public string JobId { get; init; } = string.Empty; + + [Required] + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphJobQueryType JobType { get; init; } + + [Required] + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphJobStatus Status { get; init; } + + [Required] + public DateTimeOffset OccurredAt { get; init; } + + public string? GraphSnapshotId { get; init; } + + public string? ResultUri { get; init; } + + public string? CorrelationId { get; init; } + + public string? Error { get; init; } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobEndpointExtensions.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobEndpointExtensions.cs new file mode 100644 index 00000000..b9b261ad --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobEndpointExtensions.cs @@ -0,0 +1,161 @@ +using Microsoft.AspNetCore.Mvc; +using System.ComponentModel.DataAnnotations; +using StellaOps.Auth.Abstractions; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.WebService.Auth; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public static class GraphJobEndpointExtensions +{ + public static void MapGraphJobEndpoints(this IEndpointRouteBuilder builder) + { + var group = builder.MapGroup("/graphs"); + + group.MapPost("/build", CreateGraphBuildJob); + group.MapPost("/overlays", CreateGraphOverlayJob); + group.MapGet("/jobs", GetGraphJobs); + group.MapPost("/hooks/completed", CompleteGraphJob); + group.MapGet("/overlays/lag", GetOverlayLagMetrics); + } + + internal static async Task CreateGraphBuildJob( + [FromBody] GraphBuildJobRequest request, + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer authorizer, + [FromServices] IGraphJobService jobService, + CancellationToken cancellationToken) + { + try + { + authorizer.EnsureScope(httpContext, StellaOpsScopes.GraphWrite); + var tenant = tenantAccessor.GetTenant(httpContext); + var job = await jobService.CreateBuildJobAsync(tenant.TenantId, request, cancellationToken); + return Results.Created($"/graphs/jobs/{job.Id}", GraphJobResponse.From(job)); + } + catch (UnauthorizedAccessException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized); + } + catch (InvalidOperationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden); + } + catch (ValidationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status400BadRequest); + } + catch (KeyNotFoundException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status404NotFound); + } + } + + internal static async Task CreateGraphOverlayJob( + [FromBody] GraphOverlayJobRequest request, + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer authorizer, + [FromServices] IGraphJobService jobService, + CancellationToken cancellationToken) + { + try + { + authorizer.EnsureScope(httpContext, StellaOpsScopes.GraphWrite); + var tenant = tenantAccessor.GetTenant(httpContext); + var job = await jobService.CreateOverlayJobAsync(tenant.TenantId, request, cancellationToken); + return Results.Created($"/graphs/jobs/{job.Id}", GraphJobResponse.From(job)); + } + catch (UnauthorizedAccessException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized); + } + catch (InvalidOperationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden); + } + catch (ValidationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status400BadRequest); + } + } + + internal static async Task GetGraphJobs( + [AsParameters] GraphJobQuery query, + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer authorizer, + [FromServices] IGraphJobService jobService, + CancellationToken cancellationToken) + { + try + { + authorizer.EnsureScope(httpContext, StellaOpsScopes.GraphRead); + var tenant = tenantAccessor.GetTenant(httpContext); + var jobs = await jobService.GetJobsAsync(tenant.TenantId, query, cancellationToken); + return Results.Ok(jobs); + } + catch (UnauthorizedAccessException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized); + } + catch (InvalidOperationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden); + } + } + + internal static async Task CompleteGraphJob( + [FromBody] GraphJobCompletionRequest request, + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer authorizer, + [FromServices] IGraphJobService jobService, + CancellationToken cancellationToken) + { + try + { + authorizer.EnsureScope(httpContext, StellaOpsScopes.GraphWrite); + var tenant = tenantAccessor.GetTenant(httpContext); + var response = await jobService.CompleteJobAsync(tenant.TenantId, request, cancellationToken); + return Results.Ok(response); + } + catch (UnauthorizedAccessException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized); + } + catch (KeyNotFoundException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status404NotFound); + } + catch (InvalidOperationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status403Forbidden); + } + catch (ValidationException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status400BadRequest); + } + } + + internal static async Task GetOverlayLagMetrics( + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer authorizer, + [FromServices] IGraphJobService jobService, + CancellationToken cancellationToken) + { + try + { + authorizer.EnsureScope(httpContext, StellaOpsScopes.GraphRead); + var tenant = tenantAccessor.GetTenant(httpContext); + var metrics = await jobService.GetOverlayLagMetricsAsync(tenant.TenantId, cancellationToken); + return Results.Ok(metrics); + } + catch (UnauthorizedAccessException ex) + { + return Results.Json(new { error = ex.Message }, statusCode: StatusCodes.Status401Unauthorized); + } + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobQuery.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobQuery.cs new file mode 100644 index 00000000..8d03e2a6 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobQuery.cs @@ -0,0 +1,27 @@ +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record GraphJobQuery +{ + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphJobQueryType? Type { get; init; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphJobStatus? Status { get; init; } + + public int? Limit { get; init; } + + internal GraphJobQuery Normalize() + => this with + { + Limit = Limit is null or <= 0 or > 200 ? 50 : Limit + }; +} + +public enum GraphJobQueryType +{ + Build, + Overlay +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobResponse.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobResponse.cs new file mode 100644 index 00000000..d692bb39 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobResponse.cs @@ -0,0 +1,45 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record GraphJobResponse +{ + public required string Id { get; init; } + public required string TenantId { get; init; } + public required string Kind { get; init; } + public required GraphJobStatus Status { get; init; } + public required object Payload { get; init; } + + public static GraphJobResponse From(GraphBuildJob job) + => new() + { + Id = job.Id, + TenantId = job.TenantId, + Kind = "build", + Status = job.Status, + Payload = job + }; + + public static GraphJobResponse From(GraphOverlayJob job) + => new() + { + Id = job.Id, + TenantId = job.TenantId, + Kind = "overlay", + Status = job.Status, + Payload = job + }; +} + +public sealed record GraphJobCollection(IReadOnlyList Jobs) +{ + public static GraphJobCollection From(IEnumerable builds, IEnumerable overlays) + { + var responses = builds.Select(GraphJobResponse.From) + .Concat(overlays.Select(GraphJobResponse.From)) + .OrderBy(response => response.Id, StringComparer.Ordinal) + .ToArray(); + + return new GraphJobCollection(responses); + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobService.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobService.cs new file mode 100644 index 00000000..6e4733b7 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphJobService.cs @@ -0,0 +1,338 @@ +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +internal sealed class GraphJobService : IGraphJobService +{ + private readonly IGraphJobStore _store; + private readonly ISystemClock _clock; + private readonly IGraphJobCompletionPublisher _completionPublisher; + private readonly ICartographerWebhookClient _cartographerWebhook; + + public GraphJobService( + IGraphJobStore store, + ISystemClock clock, + IGraphJobCompletionPublisher completionPublisher, + ICartographerWebhookClient cartographerWebhook) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + _clock = clock ?? throw new ArgumentNullException(nameof(clock)); + _completionPublisher = completionPublisher ?? throw new ArgumentNullException(nameof(completionPublisher)); + _cartographerWebhook = cartographerWebhook ?? throw new ArgumentNullException(nameof(cartographerWebhook)); + } + + public async Task CreateBuildJobAsync(string tenantId, GraphBuildJobRequest request, CancellationToken cancellationToken) + { + Validate(request); + + var trigger = request.Trigger ?? GraphBuildJobTrigger.SbomVersion; + var metadata = request.Metadata ?? new Dictionary(StringComparer.Ordinal); + + var now = _clock.UtcNow; + var id = GenerateIdentifier("gbj"); + var job = new GraphBuildJob( + id, + tenantId, + request.SbomId.Trim(), + request.SbomVersionId.Trim(), + NormalizeDigest(request.SbomDigest), + GraphJobStatus.Pending, + trigger, + now, + request.GraphSnapshotId, + attempts: 0, + cartographerJobId: null, + correlationId: request.CorrelationId?.Trim(), + startedAt: null, + completedAt: null, + error: null, + metadata: metadata.Select(pair => new KeyValuePair(pair.Key, pair.Value))); + + return await _store.AddAsync(job, cancellationToken); + } + + public async Task CreateOverlayJobAsync(string tenantId, GraphOverlayJobRequest request, CancellationToken cancellationToken) + { + Validate(request); + + var subjects = (request.Subjects ?? Array.Empty()) + .Where(subject => !string.IsNullOrWhiteSpace(subject)) + .Select(subject => subject.Trim()) + .ToArray(); + var metadata = request.Metadata ?? new Dictionary(StringComparer.Ordinal); + var trigger = request.Trigger ?? GraphOverlayJobTrigger.Policy; + + var now = _clock.UtcNow; + var id = GenerateIdentifier("goj"); + + var job = new GraphOverlayJob( + id: id, + tenantId: tenantId, + graphSnapshotId: request.GraphSnapshotId.Trim(), + overlayKind: request.OverlayKind, + overlayKey: request.OverlayKey.Trim(), + status: GraphJobStatus.Pending, + trigger: trigger, + createdAt: now, + subjects: subjects, + attempts: 0, + buildJobId: request.BuildJobId?.Trim(), + correlationId: request.CorrelationId?.Trim(), + metadata: metadata.Select(pair => new KeyValuePair(pair.Key, pair.Value))); + + return await _store.AddAsync(job, cancellationToken); + } + + public async Task GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken) + { + return await _store.GetJobsAsync(tenantId, query, cancellationToken); + } + + public async Task CompleteJobAsync(string tenantId, GraphJobCompletionRequest request, CancellationToken cancellationToken) + { + if (request.Status is not (GraphJobStatus.Completed or GraphJobStatus.Failed or GraphJobStatus.Cancelled)) + { + throw new ValidationException("Completion requires status completed, failed, or cancelled."); + } + + var occurredAt = request.OccurredAt == default ? _clock.UtcNow : request.OccurredAt.ToUniversalTime(); + + switch (request.JobType) + { + case GraphJobQueryType.Build: + { + var existing = await _store.GetBuildJobAsync(tenantId, request.JobId, cancellationToken); + if (existing is null) + { + throw new KeyNotFoundException($"Graph build job '{request.JobId}' not found."); + } + + var current = existing; + if (current.Status is GraphJobStatus.Pending or GraphJobStatus.Queued) + { + current = GraphJobStateMachine.EnsureTransition(current, GraphJobStatus.Running, occurredAt, attempts: current.Attempts); + } + + var updated = GraphJobStateMachine.EnsureTransition(current, request.Status, occurredAt, attempts: current.Attempts + 1, errorMessage: request.Error); + var metadata = MergeMetadata(updated.Metadata, request.ResultUri); + var normalized = new GraphBuildJob( + id: updated.Id, + tenantId: updated.TenantId, + sbomId: updated.SbomId, + sbomVersionId: updated.SbomVersionId, + sbomDigest: updated.SbomDigest, + graphSnapshotId: request.GraphSnapshotId?.Trim() ?? updated.GraphSnapshotId, + status: updated.Status, + trigger: updated.Trigger, + attempts: updated.Attempts, + cartographerJobId: updated.CartographerJobId, + correlationId: request.CorrelationId?.Trim() ?? updated.CorrelationId, + createdAt: updated.CreatedAt, + startedAt: updated.StartedAt, + completedAt: updated.CompletedAt, + error: updated.Error, + metadata: metadata, + schemaVersion: updated.SchemaVersion); + + var stored = await _store.UpdateAsync(normalized, cancellationToken); + var response = GraphJobResponse.From(stored); + await PublishCompletionAsync(tenantId, GraphJobQueryType.Build, request.Status, occurredAt, response, request.ResultUri, request.CorrelationId, request.Error, cancellationToken); + return response; + } + + case GraphJobQueryType.Overlay: + { + var existing = await _store.GetOverlayJobAsync(tenantId, request.JobId, cancellationToken); + if (existing is null) + { + throw new KeyNotFoundException($"Graph overlay job '{request.JobId}' not found."); + } + + var current = existing; + if (current.Status is GraphJobStatus.Pending or GraphJobStatus.Queued) + { + current = GraphJobStateMachine.EnsureTransition(current, GraphJobStatus.Running, occurredAt, attempts: current.Attempts); + } + + var updated = GraphJobStateMachine.EnsureTransition(current, request.Status, occurredAt, attempts: current.Attempts + 1, errorMessage: request.Error); + var metadata = MergeMetadata(updated.Metadata, request.ResultUri); + var normalized = new GraphOverlayJob( + id: updated.Id, + tenantId: updated.TenantId, + graphSnapshotId: updated.GraphSnapshotId, + buildJobId: updated.BuildJobId, + overlayKind: updated.OverlayKind, + overlayKey: updated.OverlayKey, + subjects: updated.Subjects, + status: updated.Status, + trigger: updated.Trigger, + attempts: updated.Attempts, + correlationId: request.CorrelationId?.Trim() ?? updated.CorrelationId, + createdAt: updated.CreatedAt, + startedAt: updated.StartedAt, + completedAt: updated.CompletedAt, + error: updated.Error, + metadata: metadata, + schemaVersion: updated.SchemaVersion); + + var stored = await _store.UpdateAsync(normalized, cancellationToken); + var response = GraphJobResponse.From(stored); + await PublishCompletionAsync(tenantId, GraphJobQueryType.Overlay, request.Status, occurredAt, response, request.ResultUri, request.CorrelationId, request.Error, cancellationToken); + return response; + } + + default: + throw new ValidationException("Unsupported job type."); + } + } + + public async Task GetOverlayLagMetricsAsync(string tenantId, CancellationToken cancellationToken) + { + var now = _clock.UtcNow; + var overlayJobs = await _store.GetOverlayJobsAsync(tenantId, cancellationToken); + + var pending = overlayJobs.Count(job => job.Status == GraphJobStatus.Pending); + var running = overlayJobs.Count(job => job.Status == GraphJobStatus.Running || job.Status == GraphJobStatus.Queued); + var completed = overlayJobs.Count(job => job.Status == GraphJobStatus.Completed); + var failed = overlayJobs.Count(job => job.Status == GraphJobStatus.Failed); + var cancelled = overlayJobs.Count(job => job.Status == GraphJobStatus.Cancelled); + + var completedJobs = overlayJobs + .Where(job => job.Status == GraphJobStatus.Completed && job.CompletedAt is not null) + .OrderByDescending(job => job.CompletedAt) + .ToArray(); + + double? minLag = null; + double? maxLag = null; + double? avgLag = null; + List recent = new(); + + if (completedJobs.Length > 0) + { + var lags = completedJobs + .Select(job => (now - job.CompletedAt!.Value).TotalSeconds) + .ToArray(); + + minLag = lags.Min(); + maxLag = lags.Max(); + avgLag = lags.Average(); + + recent = completedJobs + .Take(5) + .Select(job => new OverlayLagEntry( + JobId: job.Id, + CompletedAt: job.CompletedAt!.Value, + LagSeconds: (now - job.CompletedAt!.Value).TotalSeconds, + CorrelationId: job.CorrelationId, + ResultUri: job.Metadata.TryGetValue("resultUri", out var value) ? value : null)) + .ToList(); + } + + return new OverlayLagMetricsResponse( + TenantId: tenantId, + Pending: pending, + Running: running, + Completed: completed, + Failed: failed, + Cancelled: cancelled, + MinLagSeconds: minLag, + MaxLagSeconds: maxLag, + AverageLagSeconds: avgLag, + RecentCompleted: recent); + } + + private static void Validate(GraphBuildJobRequest request) + { + if (string.IsNullOrWhiteSpace(request.SbomId)) + { + throw new ValidationException("sbomId is required."); + } + + if (string.IsNullOrWhiteSpace(request.SbomVersionId)) + { + throw new ValidationException("sbomVersionId is required."); + } + + if (string.IsNullOrWhiteSpace(request.SbomDigest)) + { + throw new ValidationException("sbomDigest is required."); + } + } + + private static void Validate(GraphOverlayJobRequest request) + { + if (string.IsNullOrWhiteSpace(request.GraphSnapshotId)) + { + throw new ValidationException("graphSnapshotId is required."); + } + + if (string.IsNullOrWhiteSpace(request.OverlayKey)) + { + throw new ValidationException("overlayKey is required."); + } + } + + private static string GenerateIdentifier(string prefix) + => $"{prefix}_{Guid.CreateVersion7().ToString("n")}"; + + private static string NormalizeDigest(string value) + { + var text = value.Trim(); + if (!text.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + throw new ValidationException("sbomDigest must start with 'sha256:'."); + } + + var digest = text[7..]; + if (digest.Length != 64 || !digest.All(IsHex)) + { + throw new ValidationException("sbomDigest must contain 64 hexadecimal characters."); + } + + return $"sha256:{digest.ToLowerInvariant()}"; + } + + private static bool IsHex(char c) + => (c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); + + private static ImmutableSortedDictionary MergeMetadata(ImmutableSortedDictionary existing, string? resultUri) + { + if (string.IsNullOrWhiteSpace(resultUri)) + { + return existing; + } + + var builder = existing.ToBuilder(); + builder["resultUri"] = resultUri.Trim(); + return builder.ToImmutableSortedDictionary(StringComparer.Ordinal); + } + + private async Task PublishCompletionAsync( + string tenantId, + GraphJobQueryType jobType, + GraphJobStatus status, + DateTimeOffset occurredAt, + GraphJobResponse response, + string? resultUri, + string? correlationId, + string? error, + CancellationToken cancellationToken) + { + var notification = new GraphJobCompletionNotification( + tenantId, + jobType, + status, + occurredAt, + response, + resultUri, + correlationId, + error); + + await _completionPublisher.PublishAsync(notification, cancellationToken).ConfigureAwait(false); + await _cartographerWebhook.NotifyAsync(notification, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/GraphOverlayJobRequest.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphOverlayJobRequest.cs new file mode 100644 index 00000000..76ce1764 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/GraphOverlayJobRequest.cs @@ -0,0 +1,29 @@ +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record GraphOverlayJobRequest +{ + [Required] + public string GraphSnapshotId { get; init; } = string.Empty; + + public string? BuildJobId { get; init; } + + [Required] + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphOverlayKind OverlayKind { get; init; } + + [Required] + public string OverlayKey { get; init; } = string.Empty; + + public IReadOnlyList? Subjects { get; init; } + + [JsonConverter(typeof(JsonStringEnumConverter))] + public GraphOverlayJobTrigger? Trigger { get; init; } + + public string? CorrelationId { get; init; } + + public IDictionary? Metadata { get; init; } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/ICartographerWebhookClient.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/ICartographerWebhookClient.cs new file mode 100644 index 00000000..bc953d3d --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/ICartographerWebhookClient.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public interface ICartographerWebhookClient +{ + Task NotifyAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobCompletionPublisher.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobCompletionPublisher.cs new file mode 100644 index 00000000..8f036672 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobCompletionPublisher.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public interface IGraphJobCompletionPublisher +{ + Task PublishAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobService.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobService.cs new file mode 100644 index 00000000..708d83a5 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobService.cs @@ -0,0 +1,16 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public interface IGraphJobService +{ + Task CreateBuildJobAsync(string tenantId, GraphBuildJobRequest request, CancellationToken cancellationToken); + + Task CreateOverlayJobAsync(string tenantId, GraphOverlayJobRequest request, CancellationToken cancellationToken); + + Task GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken); + + Task CompleteJobAsync(string tenantId, GraphJobCompletionRequest request, CancellationToken cancellationToken); + + Task GetOverlayLagMetricsAsync(string tenantId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobStore.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobStore.cs new file mode 100644 index 00000000..6f01dd3c --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/IGraphJobStore.cs @@ -0,0 +1,22 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public interface IGraphJobStore +{ + ValueTask AddAsync(GraphBuildJob job, CancellationToken cancellationToken); + + ValueTask AddAsync(GraphOverlayJob job, CancellationToken cancellationToken); + + ValueTask GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken); + + ValueTask GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken); + + ValueTask GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken); + + ValueTask UpdateAsync(GraphBuildJob job, CancellationToken cancellationToken); + + ValueTask UpdateAsync(GraphOverlayJob job, CancellationToken cancellationToken); + + ValueTask> GetOverlayJobsAsync(string tenantId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/InMemoryGraphJobStore.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/InMemoryGraphJobStore.cs new file mode 100644 index 00000000..00f74232 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/InMemoryGraphJobStore.cs @@ -0,0 +1,83 @@ +using System.Collections.Concurrent; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +internal sealed class InMemoryGraphJobStore : IGraphJobStore +{ + private readonly ConcurrentDictionary _buildJobs = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _overlayJobs = new(StringComparer.Ordinal); + + public ValueTask AddAsync(GraphBuildJob job, CancellationToken cancellationToken) + { + _buildJobs[job.Id] = job; + return ValueTask.FromResult(job); + } + + public ValueTask AddAsync(GraphOverlayJob job, CancellationToken cancellationToken) + { + _overlayJobs[job.Id] = job; + return ValueTask.FromResult(job); + } + + public ValueTask GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken) + { + var normalized = query.Normalize(); + var buildJobs = _buildJobs.Values + .Where(job => string.Equals(job.TenantId, tenantId, StringComparison.Ordinal)) + .Where(job => normalized.Status is null || job.Status == normalized.Status) + .OrderByDescending(job => job.CreatedAt) + .Take(normalized.Limit ?? 50) + .ToArray(); + + var overlayJobs = _overlayJobs.Values + .Where(job => string.Equals(job.TenantId, tenantId, StringComparison.Ordinal)) + .Where(job => normalized.Status is null || job.Status == normalized.Status) + .OrderByDescending(job => job.CreatedAt) + .Take(normalized.Limit ?? 50) + .ToArray(); + + return ValueTask.FromResult(GraphJobCollection.From(buildJobs, overlayJobs)); + } + + public ValueTask GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken) + { + if (_buildJobs.TryGetValue(jobId, out var job) && string.Equals(job.TenantId, tenantId, StringComparison.Ordinal)) + { + return ValueTask.FromResult(job); + } + + return ValueTask.FromResult(null); + } + + public ValueTask GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken) + { + if (_overlayJobs.TryGetValue(jobId, out var job) && string.Equals(job.TenantId, tenantId, StringComparison.Ordinal)) + { + return ValueTask.FromResult(job); + } + + return ValueTask.FromResult(null); + } + + public ValueTask UpdateAsync(GraphBuildJob job, CancellationToken cancellationToken) + { + _buildJobs[job.Id] = job; + return ValueTask.FromResult(job); + } + + public ValueTask UpdateAsync(GraphOverlayJob job, CancellationToken cancellationToken) + { + _overlayJobs[job.Id] = job; + return ValueTask.FromResult(job); + } + + public ValueTask> GetOverlayJobsAsync(string tenantId, CancellationToken cancellationToken) + { + var jobs = _overlayJobs.Values + .Where(job => string.Equals(job.TenantId, tenantId, StringComparison.Ordinal)) + .ToArray(); + + return ValueTask.FromResult>(jobs); + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/MongoGraphJobStore.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/MongoGraphJobStore.cs new file mode 100644 index 00000000..952c8b52 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/MongoGraphJobStore.cs @@ -0,0 +1,55 @@ +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +internal sealed class MongoGraphJobStore : IGraphJobStore +{ + private readonly IGraphJobRepository _repository; + + public MongoGraphJobStore(IGraphJobRepository repository) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + } + + public async ValueTask AddAsync(GraphBuildJob job, CancellationToken cancellationToken) + { + await _repository.InsertAsync(job, cancellationToken); + return job; + } + + public async ValueTask AddAsync(GraphOverlayJob job, CancellationToken cancellationToken) + { + await _repository.InsertAsync(job, cancellationToken); + return job; + } + + public async ValueTask GetJobsAsync(string tenantId, GraphJobQuery query, CancellationToken cancellationToken) + { + var normalized = query.Normalize(); + var builds = normalized.Type is null or GraphJobQueryType.Build + ? await _repository.ListBuildJobsAsync(tenantId, normalized.Status, normalized.Limit ?? 50, cancellationToken) + : Array.Empty(); + + var overlays = normalized.Type is null or GraphJobQueryType.Overlay + ? await _repository.ListOverlayJobsAsync(tenantId, normalized.Status, normalized.Limit ?? 50, cancellationToken) + : Array.Empty(); + + return GraphJobCollection.From(builds, overlays); + } + + public async ValueTask GetBuildJobAsync(string tenantId, string jobId, CancellationToken cancellationToken) + => await _repository.GetBuildJobAsync(tenantId, jobId, cancellationToken); + + public async ValueTask GetOverlayJobAsync(string tenantId, string jobId, CancellationToken cancellationToken) + => await _repository.GetOverlayJobAsync(tenantId, jobId, cancellationToken); + + public async ValueTask UpdateAsync(GraphBuildJob job, CancellationToken cancellationToken) + => await _repository.ReplaceAsync(job, cancellationToken); + + public async ValueTask UpdateAsync(GraphOverlayJob job, CancellationToken cancellationToken) + => await _repository.ReplaceAsync(job, cancellationToken); + + public async ValueTask> GetOverlayJobsAsync(string tenantId, CancellationToken cancellationToken) + => await _repository.ListOverlayJobsAsync(tenantId, cancellationToken); +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/NullCartographerWebhookClient.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/NullCartographerWebhookClient.cs new file mode 100644 index 00000000..86226432 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/NullCartographerWebhookClient.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +internal sealed class NullCartographerWebhookClient : ICartographerWebhookClient +{ + private readonly ILogger _logger; + + public NullCartographerWebhookClient(ILogger logger) + => _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + public Task NotifyAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken) + { + _logger.LogDebug("Cartographer webhook suppressed for tenant {TenantId}, job {JobId} ({Status}).", notification.TenantId, notification.Job.Id, notification.Status); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/NullGraphJobCompletionPublisher.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/NullGraphJobCompletionPublisher.cs new file mode 100644 index 00000000..e87b8744 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/NullGraphJobCompletionPublisher.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scheduler.WebService.GraphJobs; + +internal sealed class NullGraphJobCompletionPublisher : IGraphJobCompletionPublisher +{ + private readonly ILogger _logger; + + public NullGraphJobCompletionPublisher(ILogger logger) + => _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + public Task PublishAsync(GraphJobCompletionNotification notification, CancellationToken cancellationToken) + { + _logger.LogDebug("Graph job completion notification suppressed for tenant {TenantId}, job {JobId} ({Status}).", notification.TenantId, notification.Job.Id, notification.Status); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scheduler.WebService/GraphJobs/OverlayLagMetricsResponse.cs b/src/StellaOps.Scheduler.WebService/GraphJobs/OverlayLagMetricsResponse.cs new file mode 100644 index 00000000..3c2d543f --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/GraphJobs/OverlayLagMetricsResponse.cs @@ -0,0 +1,20 @@ +namespace StellaOps.Scheduler.WebService.GraphJobs; + +public sealed record OverlayLagMetricsResponse( + string TenantId, + int Pending, + int Running, + int Completed, + int Failed, + int Cancelled, + double? MinLagSeconds, + double? MaxLagSeconds, + double? AverageLagSeconds, + IReadOnlyList RecentCompleted); + +public sealed record OverlayLagEntry( + string JobId, + DateTimeOffset CompletedAt, + double LagSeconds, + string? CorrelationId, + string? ResultUri); diff --git a/src/StellaOps.Scheduler.WebService/ISystemClock.cs b/src/StellaOps.Scheduler.WebService/ISystemClock.cs new file mode 100644 index 00000000..76adff77 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/ISystemClock.cs @@ -0,0 +1,11 @@ +namespace StellaOps.Scheduler.WebService; + +public interface ISystemClock +{ + DateTimeOffset UtcNow { get; } +} + +public sealed class SystemClock : ISystemClock +{ + public DateTimeOffset UtcNow => DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Scheduler.WebService/Options/SchedulerAuthorityOptions.cs b/src/StellaOps.Scheduler.WebService/Options/SchedulerAuthorityOptions.cs new file mode 100644 index 00000000..415ad995 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Options/SchedulerAuthorityOptions.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Scheduler.WebService.Options; + +/// +/// Configuration controlling Authority-backed authentication for the Scheduler WebService. +/// +public sealed class SchedulerAuthorityOptions +{ + public bool Enabled { get; set; } = false; + + /// + /// Allows the service to run without enforcing Authority authentication (development/tests only). + /// + public bool AllowAnonymousFallback { get; set; } + + /// + /// Authority issuer URL exposed via OpenID discovery. + /// + public string Issuer { get; set; } = string.Empty; + + public bool RequireHttpsMetadata { get; set; } = true; + + public string? MetadataAddress { get; set; } + + public int BackchannelTimeoutSeconds { get; set; } = 30; + + public int TokenClockSkewSeconds { get; set; } = 60; + + public IList Audiences { get; } = new List(); + + public IList RequiredScopes { get; } = new List(); + + public IList RequiredTenants { get; } = new List(); + + public IList BypassNetworks { get; } = new List(); + + public void Validate() + { + if (!Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(Issuer)) + { + throw new InvalidOperationException("Scheduler Authority issuer must be configured when Authority is enabled."); + } + + if (!Uri.TryCreate(Issuer.Trim(), UriKind.Absolute, out var uri)) + { + throw new InvalidOperationException("Scheduler Authority issuer must be an absolute URI."); + } + + if (RequireHttpsMetadata && !uri.IsLoopback && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Scheduler Authority issuer must use HTTPS unless targeting loopback development."); + } + + if (BackchannelTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Scheduler Authority back-channel timeout must be greater than zero seconds."); + } + + if (TokenClockSkewSeconds < 0 || TokenClockSkewSeconds > 300) + { + throw new InvalidOperationException("Scheduler Authority token clock skew must be between 0 and 300 seconds."); + } + } +} diff --git a/src/StellaOps.Scheduler.WebService/Options/SchedulerCartographerOptions.cs b/src/StellaOps.Scheduler.WebService/Options/SchedulerCartographerOptions.cs new file mode 100644 index 00000000..b13d5c2d --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Options/SchedulerCartographerOptions.cs @@ -0,0 +1,19 @@ +namespace StellaOps.Scheduler.WebService.Options; + +public sealed class SchedulerCartographerOptions +{ + public CartographerWebhookOptions Webhook { get; set; } = new(); +} + +public sealed class CartographerWebhookOptions +{ + public bool Enabled { get; set; } + + public string? Endpoint { get; set; } + + public string? ApiKeyHeader { get; set; } + + public string? ApiKey { get; set; } + + public int TimeoutSeconds { get; set; } = 10; +} diff --git a/src/StellaOps.Scheduler.WebService/Options/SchedulerEventsOptions.cs b/src/StellaOps.Scheduler.WebService/Options/SchedulerEventsOptions.cs new file mode 100644 index 00000000..1c067e9a --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Options/SchedulerEventsOptions.cs @@ -0,0 +1,109 @@ +namespace StellaOps.Scheduler.WebService.Options; + +/// +/// Scheduler WebService event options (outbound + inbound). +/// +using System; + +public sealed class SchedulerEventsOptions +{ + public GraphJobEventsOptions GraphJobs { get; set; } = new(); + + public SchedulerInboundWebhooksOptions Webhooks { get; set; } = new(); +} + +public sealed class GraphJobEventsOptions +{ + /// + /// Enables emission of legacy scheduler.graph.job.completed@1 events. + /// + public bool Enabled { get; set; } +} + +public sealed class SchedulerInboundWebhooksOptions +{ + public SchedulerWebhookOptions Feedser { get; set; } = SchedulerWebhookOptions.CreateDefault("feedser"); + + public SchedulerWebhookOptions Vexer { get; set; } = SchedulerWebhookOptions.CreateDefault("vexer"); +} + +public sealed class SchedulerWebhookOptions +{ + private const string DefaultSignatureHeader = "X-Scheduler-Signature"; + + public SchedulerWebhookOptions() + { + SignatureHeader = DefaultSignatureHeader; + } + + public bool Enabled { get; set; } = true; + + /// + /// Require a client certificate to be presented (mTLS). Optional when HMAC is configured. + /// + public bool RequireClientCertificate { get; set; } + + /// + /// Shared secret (Base64 or raw text) for HMAC-SHA256 signatures. Required if is false. + /// + public string? HmacSecret { get; set; } + + /// + /// Header name carrying the webhook signature (defaults to X-Scheduler-Signature). + /// + public string SignatureHeader { get; set; } + + /// + /// Maximum number of accepted requests per sliding window. + /// + public int RateLimitRequests { get; set; } = 60; + + /// + /// Sliding window duration in seconds for the rate limiter. + /// + public int RateLimitWindowSeconds { get; set; } = 60; + + /// + /// Optional label used for logging/diagnostics; populated via . + /// + public string Name { get; set; } = string.Empty; + + public static SchedulerWebhookOptions CreateDefault(string name) + => new() + { + Name = name, + SignatureHeader = DefaultSignatureHeader, + RateLimitRequests = 120, + RateLimitWindowSeconds = 60 + }; + + public void Validate() + { + if (!Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(SignatureHeader)) + { + throw new InvalidOperationException($"Scheduler webhook '{Name}' must specify a signature header when enabled."); + } + + if (!RequireClientCertificate && string.IsNullOrWhiteSpace(HmacSecret)) + { + throw new InvalidOperationException($"Scheduler webhook '{Name}' must configure either HMAC secret or mTLS enforcement."); + } + + if (RateLimitRequests <= 0) + { + throw new InvalidOperationException($"Scheduler webhook '{Name}' must configure a positive rate limit."); + } + + if (RateLimitWindowSeconds <= 0) + { + throw new InvalidOperationException($"Scheduler webhook '{Name}' must configure a rate limit window greater than zero seconds."); + } + } + + public TimeSpan GetRateLimitWindow() => TimeSpan.FromSeconds(RateLimitWindowSeconds <= 0 ? 60 : RateLimitWindowSeconds); +} diff --git a/src/StellaOps.Scheduler.WebService/Program.cs b/src/StellaOps.Scheduler.WebService/Program.cs new file mode 100644 index 00000000..3830ef9a --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Program.cs @@ -0,0 +1,184 @@ +using System.Linq; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Scheduler.ImpactIndex; +using StellaOps.Scheduler.Storage.Mongo; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; +using StellaOps.Scheduler.WebService; +using StellaOps.Scheduler.WebService.Auth; +using StellaOps.Scheduler.WebService.EventWebhooks; +using StellaOps.Scheduler.WebService.GraphJobs; +using StellaOps.Scheduler.WebService.GraphJobs.Events; +using StellaOps.Scheduler.WebService.Schedules; +using StellaOps.Scheduler.WebService.Options; +using StellaOps.Scheduler.WebService.Runs; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddRouting(options => options.LowercaseUrls = true); +builder.Services.AddSingleton(); +builder.Services.TryAddSingleton(TimeProvider.System); + +var authorityOptions = new SchedulerAuthorityOptions(); +builder.Configuration.GetSection("Scheduler:Authority").Bind(authorityOptions); + +if (!authorityOptions.RequiredScopes.Any(scope => string.Equals(scope, StellaOpsScopes.GraphRead, StringComparison.OrdinalIgnoreCase))) +{ + authorityOptions.RequiredScopes.Add(StellaOpsScopes.GraphRead); +} + +if (!authorityOptions.RequiredScopes.Any(scope => string.Equals(scope, StellaOpsScopes.GraphWrite, StringComparison.OrdinalIgnoreCase))) +{ + authorityOptions.RequiredScopes.Add(StellaOpsScopes.GraphWrite); +} + +if (authorityOptions.Audiences.Count == 0) +{ + authorityOptions.Audiences.Add("api://scheduler"); +} + +authorityOptions.Validate(); +builder.Services.AddSingleton(authorityOptions); + +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection("Scheduler:Events")) + .PostConfigure(options => + { + options.Webhooks ??= new SchedulerInboundWebhooksOptions(); + options.Webhooks.Feedser ??= SchedulerWebhookOptions.CreateDefault("feedser"); + options.Webhooks.Vexer ??= SchedulerWebhookOptions.CreateDefault("vexer"); + + options.Webhooks.Feedser.Name = string.IsNullOrWhiteSpace(options.Webhooks.Feedser.Name) + ? "feedser" + : options.Webhooks.Feedser.Name; + options.Webhooks.Vexer.Name = string.IsNullOrWhiteSpace(options.Webhooks.Vexer.Name) + ? "vexer" + : options.Webhooks.Vexer.Name; + + options.Webhooks.Feedser.Validate(); + options.Webhooks.Vexer.Validate(); + }); + +builder.Services.AddMemoryCache(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +var cartographerOptions = builder.Configuration.GetSection("Scheduler:Cartographer").Get() ?? new SchedulerCartographerOptions(); +builder.Services.AddSingleton(cartographerOptions); +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection("Scheduler:Cartographer")); + +var storageSection = builder.Configuration.GetSection("Scheduler:Storage"); +if (storageSection.Exists()) +{ + builder.Services.AddSchedulerMongoStorage(storageSection); + builder.Services.AddSingleton(); +} +else +{ + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); +} +builder.Services.AddSingleton(); +if (cartographerOptions.Webhook.Enabled) +{ + builder.Services.AddHttpClient((serviceProvider, client) => + { + var options = serviceProvider.GetRequiredService>().CurrentValue; + client.Timeout = TimeSpan.FromSeconds(options.Webhook.TimeoutSeconds <= 0 ? 10 : options.Webhook.TimeoutSeconds); + }); +} +else +{ + builder.Services.AddSingleton(); +} +builder.Services.AddScoped(); +builder.Services.AddImpactIndexStub(); + +if (authorityOptions.Enabled) +{ + builder.Services.AddHttpContextAccessor(); + builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: null, + configure: resourceOptions => + { + resourceOptions.Authority = authorityOptions.Issuer; + resourceOptions.RequireHttpsMetadata = authorityOptions.RequireHttpsMetadata; + resourceOptions.MetadataAddress = authorityOptions.MetadataAddress; + resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(authorityOptions.BackchannelTimeoutSeconds); + resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(authorityOptions.TokenClockSkewSeconds); + + foreach (var audience in authorityOptions.Audiences) + { + resourceOptions.Audiences.Add(audience); + } + + foreach (var scope in authorityOptions.RequiredScopes) + { + resourceOptions.RequiredScopes.Add(scope); + } + + foreach (var tenant in authorityOptions.RequiredTenants) + { + resourceOptions.RequiredTenants.Add(tenant); + } + + foreach (var network in authorityOptions.BypassNetworks) + { + resourceOptions.BypassNetworks.Add(network); + } + }); + + builder.Services.AddAuthorization(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); +} +else +{ + builder.Services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = "Anonymous"; + options.DefaultChallengeScheme = "Anonymous"; + }).AddScheme("Anonymous", static _ => { }); + + builder.Services.AddAuthorization(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); +} + +builder.Services.AddEndpointsApiExplorer(); + +var app = builder.Build(); + +app.UseAuthentication(); +app.UseAuthorization(); + +if (!authorityOptions.Enabled) +{ + app.Logger.LogWarning("Scheduler Authority authentication is disabled; relying on header-based development fallback."); +} +else if (authorityOptions.AllowAnonymousFallback) +{ + app.Logger.LogWarning("Scheduler Authority authentication is enabled but anonymous fallback remains allowed. Disable fallback before production rollout."); +} + +app.MapGet("/healthz", () => Results.Json(new { status = "ok" })); +app.MapGet("/readyz", () => Results.Json(new { status = "ready" })); + +app.MapGraphJobEndpoints(); +app.MapScheduleEndpoints(); +app.MapRunEndpoints(); +app.MapSchedulerEventWebhookEndpoints(); + +app.Run(); + +public partial class Program; diff --git a/src/StellaOps.Scheduler.WebService/Properties/AssemblyInfo.cs b/src/StellaOps.Scheduler.WebService/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..21e0ad8f --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scheduler.WebService.Tests")] diff --git a/src/StellaOps.Scheduler.WebService/Runs/InMemoryRunRepository.cs b/src/StellaOps.Scheduler.WebService/Runs/InMemoryRunRepository.cs new file mode 100644 index 00000000..9c2eb62e --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Runs/InMemoryRunRepository.cs @@ -0,0 +1,109 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; + +namespace StellaOps.Scheduler.WebService.Runs; + +internal sealed class InMemoryRunRepository : IRunRepository +{ + private readonly ConcurrentDictionary _runs = new(StringComparer.Ordinal); + + public Task InsertAsync( + Run run, + MongoDB.Driver.IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(run); + _runs[run.Id] = run; + return Task.CompletedTask; + } + + public Task UpdateAsync( + Run run, + MongoDB.Driver.IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(run); + + if (!_runs.TryGetValue(run.Id, out var existing)) + { + return Task.FromResult(false); + } + + if (!string.Equals(existing.TenantId, run.TenantId, StringComparison.Ordinal)) + { + return Task.FromResult(false); + } + + _runs[run.Id] = run; + return Task.FromResult(true); + } + + public Task GetAsync( + string tenantId, + string runId, + MongoDB.Driver.IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + if (string.IsNullOrWhiteSpace(runId)) + { + throw new ArgumentException("Run id must be provided.", nameof(runId)); + } + + if (_runs.TryGetValue(runId, out var run) && string.Equals(run.TenantId, tenantId, StringComparison.Ordinal)) + { + return Task.FromResult(run); + } + + return Task.FromResult(null); + } + + public Task> ListAsync( + string tenantId, + RunQueryOptions? options = null, + MongoDB.Driver.IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant id must be provided.", nameof(tenantId)); + } + + options ??= new RunQueryOptions(); + + IEnumerable query = _runs.Values + .Where(run => string.Equals(run.TenantId, tenantId, StringComparison.Ordinal)); + + if (!string.IsNullOrWhiteSpace(options.ScheduleId)) + { + query = query.Where(run => string.Equals(run.ScheduleId, options.ScheduleId, StringComparison.Ordinal)); + } + + if (!options.States.IsDefaultOrEmpty) + { + var allowed = options.States.ToImmutableHashSet(); + query = query.Where(run => allowed.Contains(run.State)); + } + + if (options.CreatedAfter is { } createdAfter) + { + query = query.Where(run => run.CreatedAt > createdAfter); + } + + query = options.SortAscending + ? query.OrderBy(run => run.CreatedAt).ThenBy(run => run.Id, StringComparer.Ordinal) + : query.OrderByDescending(run => run.CreatedAt).ThenByDescending(run => run.Id, StringComparer.Ordinal); + + var limit = options.Limit is { } specified && specified > 0 ? specified : 50; + var result = query.Take(limit).ToArray(); + return Task.FromResult>(result); + } +} diff --git a/src/StellaOps.Scheduler.WebService/Runs/RunContracts.cs b/src/StellaOps.Scheduler.WebService/Runs/RunContracts.cs new file mode 100644 index 00000000..a9313b86 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Runs/RunContracts.cs @@ -0,0 +1,40 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.WebService.Runs; + +internal sealed record RunCreateRequest( + [property: JsonPropertyName("scheduleId")] string? ScheduleId, + [property: JsonPropertyName("trigger")] RunTrigger Trigger = RunTrigger.Manual, + [property: JsonPropertyName("reason")] RunReason? Reason = null, + [property: JsonPropertyName("correlationId")] string? CorrelationId = null); + +internal sealed record RunCollectionResponse( + [property: JsonPropertyName("runs")] IReadOnlyList Runs); + +internal sealed record RunResponse( + [property: JsonPropertyName("run")] Run Run); + +internal sealed record ImpactPreviewRequest( + [property: JsonPropertyName("scheduleId")] string? ScheduleId, + [property: JsonPropertyName("selector")] Selector? Selector, + [property: JsonPropertyName("productKeys")] ImmutableArray? ProductKeys, + [property: JsonPropertyName("vulnerabilityIds")] ImmutableArray? VulnerabilityIds, + [property: JsonPropertyName("usageOnly")] bool UsageOnly = true, + [property: JsonPropertyName("sampleSize")] int SampleSize = 10); + +internal sealed record ImpactPreviewResponse( + [property: JsonPropertyName("total")] int Total, + [property: JsonPropertyName("usageOnly")] bool UsageOnly, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("snapshotId")] string? SnapshotId, + [property: JsonPropertyName("sample")] ImmutableArray Sample); + +internal sealed record ImpactPreviewSample( + [property: JsonPropertyName("imageDigest")] string ImageDigest, + [property: JsonPropertyName("registry")] string Registry, + [property: JsonPropertyName("repository")] string Repository, + [property: JsonPropertyName("namespaces")] ImmutableArray Namespaces, + [property: JsonPropertyName("tags")] ImmutableArray Tags, + [property: JsonPropertyName("usedByEntrypoint")] bool UsedByEntrypoint); diff --git a/src/StellaOps.Scheduler.WebService/Runs/RunEndpoints.cs b/src/StellaOps.Scheduler.WebService/Runs/RunEndpoints.cs new file mode 100644 index 00000000..232a2492 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Runs/RunEndpoints.cs @@ -0,0 +1,419 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.Primitives; +using StellaOps.Scheduler.ImpactIndex; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; +using StellaOps.Scheduler.WebService.Auth; + +namespace StellaOps.Scheduler.WebService.Runs; + +internal static class RunEndpoints +{ + private const string ReadScope = "scheduler.runs.read"; + private const string WriteScope = "scheduler.runs.write"; + private const string PreviewScope = "scheduler.runs.preview"; + + public static IEndpointRouteBuilder MapRunEndpoints(this IEndpointRouteBuilder routes) + { + var group = routes.MapGroup("/api/v1/scheduler/runs"); + + group.MapGet("/", ListRunsAsync); + group.MapGet("/{runId}", GetRunAsync); + group.MapPost("/", CreateRunAsync); + group.MapPost("/{runId}/cancel", CancelRunAsync); + group.MapPost("/preview", PreviewImpactAsync); + + return routes; + } + + private static async Task ListRunsAsync( + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IRunRepository repository, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, ReadScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var scheduleId = httpContext.Request.Query.TryGetValue("scheduleId", out var scheduleValues) + ? scheduleValues.ToString().Trim() + : null; + + var states = ParseRunStates(httpContext.Request.Query.TryGetValue("state", out var stateValues) ? stateValues : StringValues.Empty); + var createdAfter = SchedulerEndpointHelpers.TryParseDateTimeOffset(httpContext.Request.Query.TryGetValue("createdAfter", out var createdAfterValues) ? createdAfterValues.ToString() : null); + var limit = SchedulerEndpointHelpers.TryParsePositiveInt(httpContext.Request.Query.TryGetValue("limit", out var limitValues) ? limitValues.ToString() : null); + + var sortAscending = httpContext.Request.Query.TryGetValue("sort", out var sortValues) && + sortValues.Any(value => string.Equals(value, "asc", StringComparison.OrdinalIgnoreCase)); + + var options = new RunQueryOptions + { + ScheduleId = string.IsNullOrWhiteSpace(scheduleId) ? null : scheduleId, + States = states, + CreatedAfter = createdAfter, + Limit = limit, + SortAscending = sortAscending, + }; + + var runs = await repository.ListAsync(tenant.TenantId, options, cancellationToken: cancellationToken).ConfigureAwait(false); + return Results.Ok(new RunCollectionResponse(runs)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task GetRunAsync( + HttpContext httpContext, + string runId, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IRunRepository repository, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, ReadScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var run = await repository.GetAsync(tenant.TenantId, runId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (run is null) + { + return Results.NotFound(); + } + + return Results.Ok(new RunResponse(run)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task CreateRunAsync( + HttpContext httpContext, + RunCreateRequest request, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository scheduleRepository, + [FromServices] IRunRepository runRepository, + [FromServices] IRunSummaryService runSummaryService, + [FromServices] ISchedulerAuditService auditService, + [FromServices] TimeProvider timeProvider, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, WriteScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + if (string.IsNullOrWhiteSpace(request.ScheduleId)) + { + throw new ValidationException("scheduleId must be provided when creating a run."); + } + + var scheduleId = request.ScheduleId!.Trim(); + if (scheduleId.Length == 0) + { + throw new ValidationException("scheduleId must contain a value."); + } + + var schedule = await scheduleRepository.GetAsync(tenant.TenantId, scheduleId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (schedule is null) + { + return Results.NotFound(); + } + + if (request.Trigger != RunTrigger.Manual) + { + throw new ValidationException("Only manual runs can be created via this endpoint."); + } + + var now = timeProvider.GetUtcNow(); + var runId = SchedulerEndpointHelpers.GenerateIdentifier("run"); + var reason = request.Reason ?? RunReason.Empty; + + var run = new Run( + runId, + tenant.TenantId, + request.Trigger, + RunState.Planning, + RunStats.Empty, + now, + reason, + schedule.Id); + + await runRepository.InsertAsync(run, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(run.ScheduleId)) + { + await runSummaryService.ProjectAsync(run, cancellationToken).ConfigureAwait(false); + } + + await auditService.WriteAsync( + new SchedulerAuditEvent( + tenant.TenantId, + "scheduler.run", + "create", + SchedulerEndpointHelpers.ResolveAuditActor(httpContext), + RunId: run.Id, + ScheduleId: schedule.Id, + Metadata: BuildMetadata( + ("state", run.State.ToString().ToLowerInvariant()), + ("trigger", run.Trigger.ToString().ToLowerInvariant()), + ("correlationId", request.CorrelationId?.Trim()))), + cancellationToken).ConfigureAwait(false); + + return Results.Created($"/api/v1/scheduler/runs/{run.Id}", new RunResponse(run)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task CancelRunAsync( + HttpContext httpContext, + string runId, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IRunRepository repository, + [FromServices] IRunSummaryService runSummaryService, + [FromServices] ISchedulerAuditService auditService, + [FromServices] TimeProvider timeProvider, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, WriteScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var run = await repository.GetAsync(tenant.TenantId, runId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (run is null) + { + return Results.NotFound(); + } + + if (RunStateMachine.IsTerminal(run.State)) + { + return Results.Conflict(new { error = "Run is already in a terminal state." }); + } + + var now = timeProvider.GetUtcNow(); + var cancelled = RunStateMachine.EnsureTransition(run, RunState.Cancelled, now); + var updated = await repository.UpdateAsync(cancelled, cancellationToken: cancellationToken).ConfigureAwait(false); + if (!updated) + { + return Results.Conflict(new { error = "Run could not be updated because it changed concurrently." }); + } + + if (!string.IsNullOrWhiteSpace(cancelled.ScheduleId)) + { + await runSummaryService.ProjectAsync(cancelled, cancellationToken).ConfigureAwait(false); + } + + await auditService.WriteAsync( + new SchedulerAuditEvent( + tenant.TenantId, + "scheduler.run", + "cancel", + SchedulerEndpointHelpers.ResolveAuditActor(httpContext), + RunId: cancelled.Id, + ScheduleId: cancelled.ScheduleId, + Metadata: BuildMetadata(("state", cancelled.State.ToString().ToLowerInvariant()))), + cancellationToken).ConfigureAwait(false); + + return Results.Ok(new RunResponse(cancelled)); + } + catch (InvalidOperationException ex) + { + return Results.BadRequest(new { error = ex.Message }); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task PreviewImpactAsync( + HttpContext httpContext, + ImpactPreviewRequest request, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository scheduleRepository, + [FromServices] IImpactIndex impactIndex, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, PreviewScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var selector = await ResolveSelectorAsync(request, tenant.TenantId, scheduleRepository, cancellationToken).ConfigureAwait(false); + + var normalizedProductKeys = NormalizeStringInputs(request.ProductKeys); + var normalizedVulnerabilityIds = NormalizeStringInputs(request.VulnerabilityIds); + + ImpactSet impactSet; + if (!normalizedProductKeys.IsDefaultOrEmpty) + { + impactSet = await impactIndex.ResolveByPurlsAsync(normalizedProductKeys, request.UsageOnly, selector, cancellationToken).ConfigureAwait(false); + } + else if (!normalizedVulnerabilityIds.IsDefaultOrEmpty) + { + impactSet = await impactIndex.ResolveByVulnerabilitiesAsync(normalizedVulnerabilityIds, request.UsageOnly, selector, cancellationToken).ConfigureAwait(false); + } + else + { + impactSet = await impactIndex.ResolveAllAsync(selector, request.UsageOnly, cancellationToken).ConfigureAwait(false); + } + + var sampleSize = Math.Clamp(request.SampleSize, 1, 50); + var sampleBuilder = ImmutableArray.CreateBuilder(); + foreach (var image in impactSet.Images.Take(sampleSize)) + { + sampleBuilder.Add(new ImpactPreviewSample( + image.ImageDigest, + image.Registry, + image.Repository, + image.Namespaces.IsDefault ? ImmutableArray.Empty : image.Namespaces, + image.Tags.IsDefault ? ImmutableArray.Empty : image.Tags, + image.UsedByEntrypoint)); + } + + var response = new ImpactPreviewResponse( + impactSet.Total, + impactSet.UsageOnly, + impactSet.GeneratedAt, + impactSet.SnapshotId, + sampleBuilder.ToImmutable()); + + return Results.Ok(response); + } + catch (KeyNotFoundException) + { + return Results.NotFound(); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static ImmutableArray ParseRunStates(StringValues values) + { + if (values.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + if (!Enum.TryParse(value, ignoreCase: true, out var parsed)) + { + throw new ValidationException($"State '{value}' is not a valid run state."); + } + + builder.Add(parsed); + } + + return builder.ToImmutable(); + } + + private static async Task ResolveSelectorAsync( + ImpactPreviewRequest request, + string tenantId, + IScheduleRepository scheduleRepository, + CancellationToken cancellationToken) + { + Selector? selector = null; + + if (!string.IsNullOrWhiteSpace(request.ScheduleId)) + { + var schedule = await scheduleRepository.GetAsync(tenantId, request.ScheduleId!, cancellationToken: cancellationToken).ConfigureAwait(false); + if (schedule is null) + { + throw new KeyNotFoundException($"Schedule '{request.ScheduleId}' was not found for tenant '{tenantId}'."); + } + + selector = schedule.Selection; + } + + if (request.Selector is not null) + { + if (selector is not null && request.ScheduleId is not null) + { + throw new ValidationException("selector cannot be combined with scheduleId in the same request."); + } + + selector = request.Selector; + } + + if (selector is null) + { + throw new ValidationException("Either scheduleId or selector must be provided."); + } + + return SchedulerEndpointHelpers.NormalizeSelector(selector, tenantId); + } + + private static ImmutableArray NormalizeStringInputs(ImmutableArray? values) + { + if (values is null || values.Value.IsDefaultOrEmpty) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var value in values.Value) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + var trimmed = value.Trim(); + if (seen.Add(trimmed)) + { + builder.Add(trimmed); + } + } + + return builder.ToImmutable(); + } + + private static IReadOnlyDictionary BuildMetadata(params (string Key, string? Value)[] pairs) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var (key, value) in pairs) + { + if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(value)) + { + continue; + } + + metadata[key] = value!; + } + + return metadata; + } +} diff --git a/src/StellaOps.Scheduler.WebService/SchedulerEndpointHelpers.cs b/src/StellaOps.Scheduler.WebService/SchedulerEndpointHelpers.cs new file mode 100644 index 00000000..cccbd571 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/SchedulerEndpointHelpers.cs @@ -0,0 +1,127 @@ +using System.ComponentModel.DataAnnotations; +using System.Globalization; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Services; + +namespace StellaOps.Scheduler.WebService; + +internal static class SchedulerEndpointHelpers +{ + private const string ActorHeader = "X-Actor-Id"; + private const string ActorNameHeader = "X-Actor-Name"; + private const string ActorKindHeader = "X-Actor-Kind"; + private const string TenantHeader = "X-Tenant-Id"; + + public static string GenerateIdentifier(string prefix) + { + if (string.IsNullOrWhiteSpace(prefix)) + { + throw new ArgumentException("Prefix must be provided.", nameof(prefix)); + } + + return $"{prefix.Trim()}_{Guid.NewGuid():N}"; + } + + public static string ResolveActorId(HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (context.Request.Headers.TryGetValue(ActorHeader, out var values)) + { + var actor = values.ToString().Trim(); + if (!string.IsNullOrEmpty(actor)) + { + return actor; + } + } + + if (context.Request.Headers.TryGetValue(TenantHeader, out var tenant)) + { + var tenantId = tenant.ToString().Trim(); + if (!string.IsNullOrEmpty(tenantId)) + { + return tenantId; + } + } + + return "system"; + } + + public static AuditActor ResolveAuditActor(HttpContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var actorId = context.Request.Headers.TryGetValue(ActorHeader, out var idHeader) + ? idHeader.ToString().Trim() + : null; + + var displayName = context.Request.Headers.TryGetValue(ActorNameHeader, out var nameHeader) + ? nameHeader.ToString().Trim() + : null; + + var kind = context.Request.Headers.TryGetValue(ActorKindHeader, out var kindHeader) + ? kindHeader.ToString().Trim() + : null; + + if (string.IsNullOrWhiteSpace(actorId)) + { + actorId = context.Request.Headers.TryGetValue(TenantHeader, out var tenantHeader) + ? tenantHeader.ToString().Trim() + : "system"; + } + + displayName = string.IsNullOrWhiteSpace(displayName) ? actorId : displayName; + kind = string.IsNullOrWhiteSpace(kind) ? "user" : kind; + + return new AuditActor(actorId!, displayName!, kind!); + } + + public static bool TryParseBoolean(string? value) + => !string.IsNullOrWhiteSpace(value) && + (string.Equals(value, "true", StringComparison.OrdinalIgnoreCase) || + string.Equals(value, "1", StringComparison.OrdinalIgnoreCase)); + + public static int? TryParsePositiveInt(string? value) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) && parsed > 0) + { + return parsed; + } + + return null; + } + + public static DateTimeOffset? TryParseDateTimeOffset(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + throw new ValidationException($"Value '{value}' is not a valid ISO-8601 timestamp."); + } + + public static Selector NormalizeSelector(Selector selection, string tenantId) + { + ArgumentNullException.ThrowIfNull(selection); + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant identifier must be provided.", nameof(tenantId)); + } + + return new Selector( + selection.Scope, + tenantId, + selection.Namespaces, + selection.Repositories, + selection.Digests, + selection.IncludeTags, + selection.Labels, + selection.ResolvesTags); + } +} diff --git a/src/StellaOps.Scheduler.WebService/Schedules/InMemorySchedulerServices.cs b/src/StellaOps.Scheduler.WebService/Schedules/InMemorySchedulerServices.cs new file mode 100644 index 00000000..e0dadcc0 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Schedules/InMemorySchedulerServices.cs @@ -0,0 +1,153 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using MongoDB.Driver; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Projections; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; + +namespace StellaOps.Scheduler.WebService.Schedules; + +internal sealed class InMemoryScheduleRepository : IScheduleRepository +{ + private readonly ConcurrentDictionary _schedules = new(StringComparer.Ordinal); + + public Task UpsertAsync( + Schedule schedule, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + _schedules[schedule.Id] = schedule; + return Task.CompletedTask; + } + + public Task GetAsync( + string tenantId, + string scheduleId, + IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (_schedules.TryGetValue(scheduleId, out var schedule) && + string.Equals(schedule.TenantId, tenantId, StringComparison.Ordinal)) + { + return Task.FromResult(schedule); + } + + return Task.FromResult(null); + } + + public Task> ListAsync( + string tenantId, + ScheduleQueryOptions? options = null, + MongoDB.Driver.IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + options ??= new ScheduleQueryOptions(); + + var query = _schedules.Values + .Where(schedule => string.Equals(schedule.TenantId, tenantId, StringComparison.Ordinal)); + + if (!options.IncludeDisabled) + { + query = query.Where(schedule => schedule.Enabled); + } + + var result = query + .OrderBy(schedule => schedule.Name, StringComparer.Ordinal) + .Take(options.Limit ?? int.MaxValue) + .ToArray(); + + return Task.FromResult>(result); + } + + public Task SoftDeleteAsync( + string tenantId, + string scheduleId, + string deletedBy, + DateTimeOffset deletedAt, + MongoDB.Driver.IClientSessionHandle? session = null, + CancellationToken cancellationToken = default) + { + if (_schedules.TryGetValue(scheduleId, out var schedule) && + string.Equals(schedule.TenantId, tenantId, StringComparison.Ordinal)) + { + _schedules.TryRemove(scheduleId, out _); + return Task.FromResult(true); + } + + return Task.FromResult(false); + } +} + +internal sealed class InMemoryRunSummaryService : IRunSummaryService +{ + private readonly ConcurrentDictionary<(string TenantId, string ScheduleId), RunSummaryProjection> _summaries = new(); + + public Task ProjectAsync(Run run, CancellationToken cancellationToken = default) + { + var scheduleId = run.ScheduleId ?? string.Empty; + var updatedAt = run.FinishedAt ?? run.StartedAt ?? run.CreatedAt; + + var counters = new RunSummaryCounters( + Total: 0, + Planning: 0, + Queued: 0, + Running: 0, + Completed: 0, + Error: 0, + Cancelled: 0, + TotalDeltas: 0, + TotalNewCriticals: 0, + TotalNewHigh: 0, + TotalNewMedium: 0, + TotalNewLow: 0); + + var projection = new RunSummaryProjection( + run.TenantId, + scheduleId, + updatedAt, + null, + ImmutableArray.Empty, + counters); + + _summaries[(run.TenantId, scheduleId)] = projection; + return Task.FromResult(projection); + } + + public Task GetAsync(string tenantId, string scheduleId, CancellationToken cancellationToken = default) + { + _summaries.TryGetValue((tenantId, scheduleId), out var projection); + return Task.FromResult(projection); + } + + public Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + var projections = _summaries.Values + .Where(summary => string.Equals(summary.TenantId, tenantId, StringComparison.Ordinal)) + .ToArray(); + return Task.FromResult>(projections); + } +} + +internal sealed class InMemorySchedulerAuditService : ISchedulerAuditService +{ + public Task WriteAsync(SchedulerAuditEvent auditEvent, CancellationToken cancellationToken = default) + { + var occurredAt = auditEvent.OccurredAt ?? DateTimeOffset.UtcNow; + var record = new AuditRecord( + auditEvent.AuditId ?? $"audit_{Guid.NewGuid():N}", + auditEvent.TenantId, + auditEvent.Category, + auditEvent.Action, + occurredAt, + auditEvent.Actor, + auditEvent.EntityId, + auditEvent.ScheduleId, + auditEvent.RunId, + auditEvent.CorrelationId, + auditEvent.Metadata?.ToImmutableSortedDictionary(StringComparer.OrdinalIgnoreCase) ?? ImmutableSortedDictionary.Empty, + auditEvent.Message); + + return Task.FromResult(record); + } +} diff --git a/src/StellaOps.Scheduler.WebService/Schedules/ScheduleContracts.cs b/src/StellaOps.Scheduler.WebService/Schedules/ScheduleContracts.cs new file mode 100644 index 00000000..efd12c5a --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Schedules/ScheduleContracts.cs @@ -0,0 +1,34 @@ +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Text.Json.Serialization; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Projections; + +namespace StellaOps.Scheduler.WebService.Schedules; + +internal sealed record ScheduleCreateRequest( + [property: JsonPropertyName("name"), Required] string Name, + [property: JsonPropertyName("cronExpression"), Required] string CronExpression, + [property: JsonPropertyName("timezone"), Required] string Timezone, + [property: JsonPropertyName("mode")] ScheduleMode Mode = ScheduleMode.AnalysisOnly, + [property: JsonPropertyName("selection"), Required] Selector Selection = null!, + [property: JsonPropertyName("onlyIf")] ScheduleOnlyIf? OnlyIf = null, + [property: JsonPropertyName("notify")] ScheduleNotify? Notify = null, + [property: JsonPropertyName("limits")] ScheduleLimits? Limits = null, + [property: JsonPropertyName("subscribers")] ImmutableArray? Subscribers = null, + [property: JsonPropertyName("enabled")] bool Enabled = true); + +internal sealed record ScheduleUpdateRequest( + [property: JsonPropertyName("name")] string? Name, + [property: JsonPropertyName("cronExpression")] string? CronExpression, + [property: JsonPropertyName("timezone")] string? Timezone, + [property: JsonPropertyName("mode")] ScheduleMode? Mode, + [property: JsonPropertyName("selection")] Selector? Selection, + [property: JsonPropertyName("onlyIf")] ScheduleOnlyIf? OnlyIf, + [property: JsonPropertyName("notify")] ScheduleNotify? Notify, + [property: JsonPropertyName("limits")] ScheduleLimits? Limits, + [property: JsonPropertyName("subscribers")] ImmutableArray? Subscribers); + +internal sealed record ScheduleCollectionResponse(IReadOnlyList Schedules); + +internal sealed record ScheduleResponse(Schedule Schedule, RunSummaryProjection? Summary); diff --git a/src/StellaOps.Scheduler.WebService/Schedules/ScheduleEndpoints.cs b/src/StellaOps.Scheduler.WebService/Schedules/ScheduleEndpoints.cs new file mode 100644 index 00000000..2cee563d --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/Schedules/ScheduleEndpoints.cs @@ -0,0 +1,397 @@ +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Storage.Mongo.Repositories; +using StellaOps.Scheduler.Storage.Mongo.Services; +using StellaOps.Scheduler.WebService.Auth; + +namespace StellaOps.Scheduler.WebService.Schedules; + +internal static class ScheduleEndpoints +{ + private const string ReadScope = "scheduler.schedules.read"; + private const string WriteScope = "scheduler.schedules.write"; + + public static IEndpointRouteBuilder MapScheduleEndpoints(this IEndpointRouteBuilder routes) + { + var group = routes.MapGroup("/api/v1/scheduler/schedules"); + + group.MapGet("/", ListSchedulesAsync); + group.MapGet("/{scheduleId}", GetScheduleAsync); + group.MapPost("/", CreateScheduleAsync); + group.MapPatch("/{scheduleId}", UpdateScheduleAsync); + group.MapPost("/{scheduleId}/pause", PauseScheduleAsync); + group.MapPost("/{scheduleId}/resume", ResumeScheduleAsync); + + return routes; + } + + private static async Task ListSchedulesAsync( + HttpContext httpContext, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository repository, + [FromServices] IRunSummaryService runSummaryService, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, ReadScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var includeDisabled = SchedulerEndpointHelpers.TryParseBoolean(httpContext.Request.Query.TryGetValue("includeDisabled", out var includeDisabledValues) ? includeDisabledValues.ToString() : null); + var includeDeleted = SchedulerEndpointHelpers.TryParseBoolean(httpContext.Request.Query.TryGetValue("includeDeleted", out var includeDeletedValues) ? includeDeletedValues.ToString() : null); + var limit = SchedulerEndpointHelpers.TryParsePositiveInt(httpContext.Request.Query.TryGetValue("limit", out var limitValues) ? limitValues.ToString() : null); + + var options = new ScheduleQueryOptions + { + IncludeDisabled = includeDisabled, + IncludeDeleted = includeDeleted, + Limit = limit + }; + + var schedules = await repository.ListAsync(tenant.TenantId, options, cancellationToken: cancellationToken).ConfigureAwait(false); + var summaries = await runSummaryService.ListAsync(tenant.TenantId, cancellationToken).ConfigureAwait(false); + var summaryLookup = summaries.ToDictionary(summary => summary.ScheduleId, summary => summary, StringComparer.Ordinal); + + var response = new ScheduleCollectionResponse( + schedules.Select(schedule => new ScheduleResponse(schedule, summaryLookup.GetValueOrDefault(schedule.Id))).ToArray()); + + return Results.Ok(response); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task GetScheduleAsync( + HttpContext httpContext, + string scheduleId, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository repository, + [FromServices] IRunSummaryService runSummaryService, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, ReadScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var schedule = await repository.GetAsync(tenant.TenantId, scheduleId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (schedule is null) + { + return Results.NotFound(); + } + + var summary = await runSummaryService.GetAsync(tenant.TenantId, scheduleId, cancellationToken).ConfigureAwait(false); + return Results.Ok(new ScheduleResponse(schedule, summary)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task CreateScheduleAsync( + HttpContext httpContext, + ScheduleCreateRequest request, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository repository, + [FromServices] ISchedulerAuditService auditService, + [FromServices] TimeProvider timeProvider, + [FromServices] ILoggerFactory loggerFactory, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, WriteScope); + ValidateRequest(request); + + var tenant = tenantAccessor.GetTenant(httpContext); + var now = timeProvider.GetUtcNow(); + + var selection = SchedulerEndpointHelpers.NormalizeSelector(request.Selection, tenant.TenantId); + var scheduleId = SchedulerEndpointHelpers.GenerateIdentifier("sch"); + + var subscribers = request.Subscribers ?? ImmutableArray.Empty; + var schedule = new Schedule( + scheduleId, + tenant.TenantId, + request.Name.Trim(), + request.Enabled, + request.CronExpression.Trim(), + request.Timezone.Trim(), + request.Mode, + selection, + request.OnlyIf ?? ScheduleOnlyIf.Default, + request.Notify ?? ScheduleNotify.Default, + request.Limits ?? ScheduleLimits.Default, + subscribers.IsDefault ? ImmutableArray.Empty : subscribers, + now, + SchedulerEndpointHelpers.ResolveActorId(httpContext), + now, + SchedulerEndpointHelpers.ResolveActorId(httpContext), + SchedulerSchemaVersions.Schedule); + + await repository.UpsertAsync(schedule, cancellationToken: cancellationToken).ConfigureAwait(false); + await auditService.WriteAsync( + new SchedulerAuditEvent( + tenant.TenantId, + "scheduler", + "create", + SchedulerEndpointHelpers.ResolveAuditActor(httpContext), + ScheduleId: schedule.Id, + Metadata: new Dictionary + { + ["cronExpression"] = schedule.CronExpression, + ["timezone"] = schedule.Timezone + }), + cancellationToken).ConfigureAwait(false); + + var response = new ScheduleResponse(schedule, null); + return Results.Created($"/api/v1/scheduler/schedules/{schedule.Id}", response); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task UpdateScheduleAsync( + HttpContext httpContext, + string scheduleId, + ScheduleUpdateRequest request, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository repository, + [FromServices] ISchedulerAuditService auditService, + [FromServices] TimeProvider timeProvider, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, WriteScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var existing = await repository.GetAsync(tenant.TenantId, scheduleId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (existing is null) + { + return Results.NotFound(); + } + + var updated = UpdateSchedule(existing, request, tenant.TenantId, timeProvider.GetUtcNow(), SchedulerEndpointHelpers.ResolveActorId(httpContext)); + await repository.UpsertAsync(updated, cancellationToken: cancellationToken).ConfigureAwait(false); + + await auditService.WriteAsync( + new SchedulerAuditEvent( + tenant.TenantId, + "scheduler", + "update", + SchedulerEndpointHelpers.ResolveAuditActor(httpContext), + ScheduleId: updated.Id, + Metadata: new Dictionary + { + ["updatedAt"] = updated.UpdatedAt.ToString("O") + }), + cancellationToken).ConfigureAwait(false); + + return Results.Ok(new ScheduleResponse(updated, null)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task PauseScheduleAsync( + HttpContext httpContext, + string scheduleId, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository repository, + [FromServices] ISchedulerAuditService auditService, + [FromServices] TimeProvider timeProvider, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, WriteScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var existing = await repository.GetAsync(tenant.TenantId, scheduleId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (existing is null) + { + return Results.NotFound(); + } + + if (!existing.Enabled) + { + return Results.Ok(new ScheduleResponse(existing, null)); + } + + var now = timeProvider.GetUtcNow(); + var updated = new Schedule( + existing.Id, + existing.TenantId, + existing.Name, + enabled: false, + existing.CronExpression, + existing.Timezone, + existing.Mode, + existing.Selection, + existing.OnlyIf, + existing.Notify, + existing.Limits, + existing.Subscribers, + existing.CreatedAt, + existing.CreatedBy, + now, + SchedulerEndpointHelpers.ResolveActorId(httpContext), + existing.SchemaVersion); + + await repository.UpsertAsync(updated, cancellationToken: cancellationToken).ConfigureAwait(false); + await auditService.WriteAsync( + new SchedulerAuditEvent( + tenant.TenantId, + "scheduler", + "pause", + SchedulerEndpointHelpers.ResolveAuditActor(httpContext), + ScheduleId: scheduleId, + Metadata: new Dictionary + { + ["enabled"] = "false" + }), + cancellationToken).ConfigureAwait(false); + + return Results.Ok(new ScheduleResponse(updated, null)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static async Task ResumeScheduleAsync( + HttpContext httpContext, + string scheduleId, + [FromServices] ITenantContextAccessor tenantAccessor, + [FromServices] IScopeAuthorizer scopeAuthorizer, + [FromServices] IScheduleRepository repository, + [FromServices] ISchedulerAuditService auditService, + [FromServices] TimeProvider timeProvider, + CancellationToken cancellationToken) + { + try + { + scopeAuthorizer.EnsureScope(httpContext, WriteScope); + var tenant = tenantAccessor.GetTenant(httpContext); + + var existing = await repository.GetAsync(tenant.TenantId, scheduleId, cancellationToken: cancellationToken).ConfigureAwait(false); + if (existing is null) + { + return Results.NotFound(); + } + + if (existing.Enabled) + { + return Results.Ok(new ScheduleResponse(existing, null)); + } + + var now = timeProvider.GetUtcNow(); + var updated = new Schedule( + existing.Id, + existing.TenantId, + existing.Name, + enabled: true, + existing.CronExpression, + existing.Timezone, + existing.Mode, + existing.Selection, + existing.OnlyIf, + existing.Notify, + existing.Limits, + existing.Subscribers, + existing.CreatedAt, + existing.CreatedBy, + now, + SchedulerEndpointHelpers.ResolveActorId(httpContext), + existing.SchemaVersion); + + await repository.UpsertAsync(updated, cancellationToken: cancellationToken).ConfigureAwait(false); + await auditService.WriteAsync( + new SchedulerAuditEvent( + tenant.TenantId, + "scheduler", + "resume", + SchedulerEndpointHelpers.ResolveAuditActor(httpContext), + ScheduleId: scheduleId, + Metadata: new Dictionary + { + ["enabled"] = "true" + }), + cancellationToken).ConfigureAwait(false); + + return Results.Ok(new ScheduleResponse(updated, null)); + } + catch (Exception ex) when (ex is ArgumentException or ValidationException) + { + return Results.BadRequest(new { error = ex.Message }); + } + } + + private static void ValidateRequest(ScheduleCreateRequest request) + { + if (request.Selection is null) + { + throw new ValidationException("Selection is required."); + } + } + + private static Schedule UpdateSchedule( + Schedule existing, + ScheduleUpdateRequest request, + string tenantId, + DateTimeOffset updatedAt, + string actor) + { + var name = request.Name?.Trim() ?? existing.Name; + var cronExpression = request.CronExpression?.Trim() ?? existing.CronExpression; + var timezone = request.Timezone?.Trim() ?? existing.Timezone; + var mode = request.Mode ?? existing.Mode; + var selection = request.Selection is null + ? existing.Selection + : SchedulerEndpointHelpers.NormalizeSelector(request.Selection, tenantId); + var onlyIf = request.OnlyIf ?? existing.OnlyIf; + var notify = request.Notify ?? existing.Notify; + var limits = request.Limits ?? existing.Limits; + var subscribers = request.Subscribers ?? existing.Subscribers; + + return new Schedule( + existing.Id, + existing.TenantId, + name, + existing.Enabled, + cronExpression, + timezone, + mode, + selection, + onlyIf, + notify, + limits, + subscribers.IsDefault ? ImmutableArray.Empty : subscribers, + existing.CreatedAt, + existing.CreatedBy, + updatedAt, + actor, + existing.SchemaVersion); + } + +} diff --git a/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj b/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj index fef56dcc..5b6c0035 100644 --- a/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj +++ b/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj @@ -1,7 +1,14 @@ - - - net10.0 - enable - enable - - + + + net10.0 + enable + enable + + + + + + + + + diff --git a/src/StellaOps.Scheduler.WebService/TASKS.md b/src/StellaOps.Scheduler.WebService/TASKS.md index ffe3605a..65e79a0f 100644 --- a/src/StellaOps.Scheduler.WebService/TASKS.md +++ b/src/StellaOps.Scheduler.WebService/TASKS.md @@ -3,23 +3,26 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCHED-WEB-16-101 | DOING (2025-10-19) | Scheduler WebService Guild | SCHED-MODELS-16-101 | Bootstrap Minimal API host with Authority OpTok + DPoP, health endpoints, plug-in discovery per architecture §§1–2. | Service boots with config validation; `/healthz`/`/readyz` pass; restart-only plug-ins enforced. | -| SCHED-WEB-16-102 | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Implement schedules CRUD (tenant-scoped) with cron validation, pause/resume, audit logging. | CRUD operations tested; invalid cron inputs rejected; audit entries persisted. | -| SCHED-WEB-16-103 | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Runs API (list/detail/cancel), ad-hoc run POST, and impact preview endpoints. | Integration tests cover run lifecycle; preview returns counts/sample; cancellation honoured. | -| SCHED-WEB-16-104 | TODO | Scheduler WebService Guild | SCHED-QUEUE-16-401, SCHED-STORAGE-16-201 | Webhook endpoints for Feedser/Vexer exports with mTLS/HMAC validation and rate limiting. | Webhooks validated via tests; invalid signatures rejected; rate limits documented. | +| SCHED-WEB-16-102 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-101 | Implement schedules CRUD (tenant-scoped) with cron validation, pause/resume, audit logging. | CRUD operations tested; invalid cron inputs rejected; audit entries persisted. | +| SCHED-WEB-16-103 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-16-102 | Runs API (list/detail/cancel), ad-hoc run POST, and impact preview endpoints. | Integration tests cover run lifecycle; preview returns counts/sample; cancellation honoured. | +| SCHED-WEB-16-104 | DONE (2025-10-27) | Scheduler WebService Guild | SCHED-QUEUE-16-401, SCHED-STORAGE-16-201 | Webhook endpoints for Feeder/Vexer exports with mTLS/HMAC validation and rate limiting. | Webhooks validated via tests; invalid signatures rejected; rate limits documented. | ## Policy Engine v2 (Sprint 20) | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCHED-WEB-20-001 | TODO | Scheduler WebService Guild, Policy Guild | SCHED-WEB-16-101, POLICY-ENGINE-20-000 | Expose policy run scheduling APIs (`POST /policy/runs`, `GET /policy/runs`) with tenant scoping and RBAC enforcement for `policy:run`. | Endpoints documented; integration tests cover run creation/status; unauthorized access blocked. | +> 2025-10-26: Use canonical request/response samples from `samples/api/scheduler/policy-*.json`; serializer contract defined in `src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-20-001-POLICY-RUNS.md`. | SCHED-WEB-20-002 | TODO | Scheduler WebService Guild | SCHED-WEB-20-001, SCHED-WORKER-20-301 | Provide simulation trigger endpoint returning diff preview metadata and job state for UI/CLI consumption. | Simulation endpoint returns deterministic diffs metadata; rate limits enforced; tests cover concurrency. | ## Graph Explorer v1 (Sprint 21) | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| SCHED-WEB-21-001 | TODO | Scheduler WebService Guild, Cartographer Guild | SCHED-WEB-16-101, SCHED-MODELS-21-001 | Expose graph build/overlay job APIs (`POST /graphs/build`, `GET /graphs/jobs`) with `graph:write`/`graph:read` enforcement and tenant scoping. | APIs documented; integration tests cover job submission/status; unauthorized requests blocked. | -| SCHED-WEB-21-002 | TODO | Scheduler WebService Guild | SCHED-WEB-21-001, CARTO-GRAPH-21-007 | Provide overlay lag metrics endpoint and webhook to notify Cartographer of job completions; include correlation IDs. | Endpoint returns metrics; webhook tested end-to-end; observability docs updated. | +| SCHED-WEB-21-001 | DONE (2025-10-26) | Scheduler WebService Guild, Cartographer Guild | SCHED-WEB-16-101, SCHED-MODELS-21-001 | Expose graph build/overlay job APIs (`POST /graphs/build`, `GET /graphs/jobs`) with `graph:write`/`graph:read` enforcement and tenant scoping. | APIs documented in `docs/SCHED-WEB-21-001-GRAPH-APIS.md`; integration tests cover submission/status; unauthorized requests blocked; scope checks now reference `StellaOpsScopes`. | +| SCHED-WEB-21-002 | DONE (2025-10-26) | Scheduler WebService Guild | SCHED-WEB-21-001, CARTO-GRAPH-21-007 | Provide overlay lag metrics endpoint and webhook to notify Cartographer of job completions; include correlation IDs. | `POST /graphs/hooks/completed` + `GET /graphs/overlays/lag` documented in `docs/SCHED-WEB-21-001-GRAPH-APIS.md`; integration tests cover completion + metrics. | +| SCHED-WEB-21-003 | DONE (2025-10-26) | Scheduler WebService Guild, Authority Core Guild | AUTH-GRAPH-21-001 | Replace temporary `X-Scopes`/`X-Tenant-Id` headers with Authority-issued OpTok verification and scope enforcement for graph endpoints. | Authentication configured via `AddStellaOpsResourceServerAuthentication`; authority scopes enforced end-to-end with `StellaOpsScopes`; header fallback limited to dev mode; tests updated. | +| SCHED-WEB-21-004 | DOING (2025-10-26) | Scheduler WebService Guild, Scheduler Storage Guild | SCHED-WEB-21-001, SCHED-STORAGE-16-201 | Persist graph job lifecycle to Mongo storage and publish `scheduler.graph.job.completed@1` events + outbound webhook to Cartographer. | Storage repositories updated; events emitted; webhook payload documented; integration tests cover storage + event flow. **Note:** Events currently log JSON envelopes while the shared platform bus is provisioned. Cartographer webhook now posts JSON payloads when configured; replace inline logging with bus publisher once the shared event transport is online. | ## StellaOps Console (Sprint 23) | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | diff --git a/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-16-103-RUN-APIS.md b/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-16-103-RUN-APIS.md new file mode 100644 index 00000000..8ef59c59 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-16-103-RUN-APIS.md @@ -0,0 +1,183 @@ +# SCHED-WEB-16-103 — Scheduler Run APIs + +> Status: 2025-10-26 — **Developer preview** (schema solid, planner/worker integration pending). + +## Endpoints + +| Method | Path | Description | Scopes | +| ------ | ---- | ----------- | ------ | +| `GET` | `/api/v1/scheduler/runs` | List runs for the current tenant (filter by schedule, state, createdAfter). | `scheduler.runs.read` | +| `GET` | `/api/v1/scheduler/runs/{runId}` | Retrieve run details. | `scheduler.runs.read` | +| `POST` | `/api/v1/scheduler/runs` | Create an ad-hoc run bound to an existing schedule. | `scheduler.runs.write` | +| `POST` | `/api/v1/scheduler/runs/{runId}/cancel` | Transition a run to `cancelled` when still in a non-terminal state. | `scheduler.runs.write` | +| `POST` | `/api/v1/scheduler/runs/preview` | Resolve impacted images using the ImpactIndex without enqueuing work. | `scheduler.runs.preview` | + +All endpoints require a tenant context (`X-Tenant-Id`) and the appropriate scheduler scopes. Development mode allows header-based auth; production deployments must rely on Authority-issued tokens (OpTok + DPoP). + +## Create Run (manual trigger) + +```http +POST /api/v1/scheduler/runs +X-Tenant-Id: tenant-alpha +Authorization: Bearer +``` + +```json +{ + "scheduleId": "sch_4f2c7d9e0a2b4c64a0e7b5f9d65c1234", + "trigger": "manual", + "reason": { + "manualReason": "Nightly backfill" + }, + "correlationId": "backfill-2025-10-26" +} +``` + +```json +HTTP/1.1 201 Created +Location: /api/v1/scheduler/runs/run_c7b4e9d2f6a04f8784a40476d8a2f771 +{ + "run": { + "schemaVersion": "scheduler.run@1", + "id": "run_c7b4e9d2f6a04f8784a40476d8a2f771", + "tenantId": "tenant-alpha", + "scheduleId": "sch_4f2c7d9e0a2b4c64a0e7b5f9d65c1234", + "trigger": "manual", + "state": "planning", + "stats": { + "candidates": 0, + "deduped": 0, + "queued": 0, + "completed": 0, + "deltas": 0, + "newCriticals": 0, + "newHigh": 0, + "newMedium": 0, + "newLow": 0 + }, + "reason": { + "manualReason": "Nightly backfill" + }, + "createdAt": "2025-10-26T03:12:45Z" + } +} +``` + +## List Runs + +```http +GET /api/v1/scheduler/runs?scheduleId=sch_4f2c7d9e0a2b4c64a0e7b5f9d65c1234&state=planning&limit=10 +``` + +```json +{ + "runs": [ + { + "schemaVersion": "scheduler.run@1", + "id": "run_c7b4e9d2f6a04f8784a40476d8a2f771", + "tenantId": "tenant-alpha", + "scheduleId": "sch_4f2c7d9e0a2b4c64a0e7b5f9d65c1234", + "trigger": "manual", + "state": "planning", + "stats": { + "candidates": 0, + "deduped": 0, + "queued": 0, + "completed": 0, + "deltas": 0, + "newCriticals": 0, + "newHigh": 0, + "newMedium": 0, + "newLow": 0 + }, + "reason": { + "manualReason": "Nightly backfill" + }, + "createdAt": "2025-10-26T03:12:45Z" + } + ] +} +``` + +## Cancel Run + +```http +POST /api/v1/scheduler/runs/run_c7b4e9d2f6a04f8784a40476d8a2f771/cancel +``` + +```json +{ + "run": { + "schemaVersion": "scheduler.run@1", + "id": "run_c7b4e9d2f6a04f8784a40476d8a2f771", + "tenantId": "tenant-alpha", + "scheduleId": "sch_4f2c7d9e0a2b4c64a0e7b5f9d65c1234", + "trigger": "manual", + "state": "cancelled", + "stats": { + "candidates": 0, + "deduped": 0, + "queued": 0, + "completed": 0, + "deltas": 0, + "newCriticals": 0, + "newHigh": 0, + "newMedium": 0, + "newLow": 0 + }, + "reason": { + "manualReason": "Nightly backfill" + }, + "createdAt": "2025-10-26T03:12:45Z", + "finishedAt": "2025-10-26T03:13:02Z" + } +} +``` + +## Impact Preview + +`/api/v1/scheduler/runs/preview` resolves impacted images via the ImpactIndex without mutating state. When `scheduleId` is provided the schedule selector is reused; callers may alternatively supply an explicit selector. + +```http +POST /api/v1/scheduler/runs/preview +``` + +```json +{ + "scheduleId": "sch_4f2c7d9e0a2b4c64a0e7b5f9d65c1234", + "usageOnly": true, + "sampleSize": 5 +} +``` + +```json +{ + "total": 128, + "usageOnly": true, + "generatedAt": "2025-10-26T03:12:47Z", + "snapshotId": "impact-snapshot-20251026", + "sample": [ + { + "imageDigest": "sha256:0b1f...", + "registry": "internal", + "repository": "service-api", + "namespaces": ["prod"], + "tags": ["prod-2025-10-01"], + "usedByEntrypoint": true + } + ] +} +``` + +### Validation rules + +* `scheduleId` is mandatory for run creation; ad-hoc selectors will be added alongside planner support. +* Cancelling a run already in a terminal state returns `409 Conflict`. +* Preview requires either `scheduleId` or `selector` (mutually exclusive). +* `sampleSize` is clamped to `1..50` to keep responses deterministic and lightweight. + +### Integration notes + +* Run creation and cancellation produce audit entries under category `scheduler.run` with correlation metadata when provided. +* The preview endpoint relies on the ImpactIndex stub in development. Production deployments must register the concrete index implementation before use. +* Planner/worker orchestration tasks will wire run creation to queueing in SCHED-WORKER-16-201/202. diff --git a/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-16-104-WEBHOOKS.md b/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-16-104-WEBHOOKS.md new file mode 100644 index 00000000..aa1f893c --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-16-104-WEBHOOKS.md @@ -0,0 +1,58 @@ +# SCHED-WEB-16-104 · Feedser/Vexer Webhook Endpoints + +## Overview + +Scheduler.WebService exposes inbound webhooks that allow Feedser and Vexer to +notify the planner when new exports are available. Each webhook validates the +payload, enforces signature requirements, and applies a per-endpoint rate +limit before queuing downstream processing. + +| Endpoint | Description | AuthZ | +|----------|-------------|-------| +| `POST /events/feedser-export` | Ingest Feedser export metadata (`exportId`, `changedProductKeys`, optional KEV & window). | HMAC `X-Scheduler-Signature` and/or mTLS client certificate | +| `POST /events/vexer-export` | Ingest Vexer export delta summary (`changedClaims`). | HMAC `X-Scheduler-Signature` and/or mTLS client certificate | + +## Security + +* Webhooks require either: + * mTLS with trusted client certificates; **or** + * an HMAC-SHA256 signature in the `X-Scheduler-Signature` header. The + signature must be computed as `sha256=` over the raw request body. +* Requests without the required signature/certificate return `401`. +* Secrets are configured under `Scheduler:Events:Webhooks:{Feedser|Vexer}:HmacSecret`. + +## Rate limiting + +* Each webhook enforces a sliding-window limit (`RateLimitRequests` over + `RateLimitWindowSeconds`). +* Requests over the limit return `429` and include a `Retry-After` header. +* Defaults: 120 requests / 60 seconds. Adjust via configuration. + +## Configuration + +``` +Scheduler: + Events: + Webhooks: + Feedser: + Enabled: true + HmacSecret: feedser-secret + RequireClientCertificate: false + RateLimitRequests: 120 + RateLimitWindowSeconds: 60 + Vexer: + Enabled: true + HmacSecret: vexer-secret + RequireClientCertificate: false +``` + +## Response envelope + +On success the webhook returns `202 Accepted` and a JSON body: + +``` +{ "status": "accepted" } +``` + +Failures return problem JSON with `error` describing the violation. + diff --git a/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-21-001-GRAPH-APIS.md b/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-21-001-GRAPH-APIS.md new file mode 100644 index 00000000..ccf7eb8b --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/docs/SCHED-WEB-21-001-GRAPH-APIS.md @@ -0,0 +1,137 @@ +# SCHED-WEB-21-001 — Graph Job APIs + +> Status: 2025-10-26 — **Complete** (developer preview) + +Minimal API endpoints for Cartographer orchestration live under `/graphs`. Authentication now relies on Authority-issued bearer tokens carrying `graph:*` scopes. For development scenarios you can disable `Scheduler:Authority:Enabled` and continue using legacy headers: + +- `X-Tenant-Id`: tenant identifier (matches Scheduler Models `tenantId`). +- `X-Scopes`: space-delimited scopes. `graph:write` is required for write operations, `graph:read` for queries. + +Example configuration (`appsettings.json` or environment overrides): + +```jsonc +{ + "Scheduler": { + "Authority": { + "Enabled": true, + "Issuer": "https://authority.stella-ops.local", + "Audiences": [ "api://scheduler" ], + "RequiredScopes": [ "graph:read", "graph:write" ] + }, + "Events": { + "GraphJobs": { + "Enabled": true + } + }, + "Cartographer": { + "Webhook": { + "Enabled": true, + "Endpoint": "https://cartographer.stella-ops.local/hooks/graph/completed", + "ApiKeyHeader": "X-StellaOps-Webhook-Key", + "ApiKey": "change-me", + "TimeoutSeconds": 10 + } + } + } +} +``` + +## Endpoints + +### `POST /graphs/build` +Creates a `GraphBuildJob` in `pending` state. + +Request body: + +```jsonc +{ + "sbomId": "sbom_alpha", + "sbomVersionId": "sbom_alpha_v1", + "sbomDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "trigger": "sbom-version", + "metadata": { + "sbomEventId": "sbom_evt_20251026" + } +} +``` + +Response: `201 Created` + +```jsonc +{ + "id": "gbj_018dc2f5902147e2b7f2ea05f5de1f3f", + "tenantId": "tenant-alpha", + "kind": "build", + "status": "pending", + "payload": { + "schemaVersion": "scheduler.graph-build-job@1", + "id": "gbj_018dc2f5902147e2b7f2ea05f5de1f3f", + "tenantId": "tenant-alpha", + "sbomId": "sbom_alpha", + "sbomVersionId": "sbom_alpha_v1", + "sbomDigest": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "status": "pending", + "trigger": "sbom-version", + "createdAt": "2025-10-26T12:00:00Z", + "metadata": { + "sbomeventid": "sbom_evt_20251026" + } + } +} +``` + +### `POST /graphs/overlays` +Creates a `GraphOverlayJob` in `pending` state. Include optional `buildJobId` and `subjects` filters. + +### `POST /graphs/hooks/completed` +Webhook invoked by Scheduler Worker once Cartographer finishes a build/overlay job. Requires `graph:write`. + +```jsonc +{ + "jobId": "goj_018dc2f5929b4f5c88ad1e43d0ab3b90", + "jobType": "Overlay", + "status": "Completed", // Completed | Failed | Cancelled + "occurredAt": "2025-10-26T12:02:45Z", + "correlationId": "corr-123", + "resultUri": "oras://cartographer/offline/tenant-alpha/graph_snap_20251026" +} +``` + +The endpoint advances the job through `running → terminal` transitions via `GraphJobStateMachine`, captures the latest correlation identifier, and stores the optional `resultUri` in metadata for downstream exports. + +### `GET /graphs/overlays/lag` +Returns per-tenant overlay lag metrics (counts, min/max/average lag seconds, and last five completions with correlation IDs + result URIs). Requires `graph:read`. + +### `GET /graphs/jobs` +Returns a combined `GraphJobCollection`. Query parameters: + +| Parameter | Description | +|-----------|-------------| +| `type` | Optional filter (`build` or `overlay`). | +| `status` | Optional filter using `GraphJobStatus`. | +| `limit` | Maximum number of results (default 50, max 200). | + +Response example: + +```jsonc +{ + "jobs": [ + { + "id": "gbj_018dc2f5902147e2b7f2ea05f5de1f3f", + "tenantId": "tenant-alpha", + "kind": "build", + "status": "pending", + "payload": { /* graph build job */ } + } + ] +} +``` + +## Integration tests + +`StellaOps.Scheduler.WebService.Tests/GraphJobEndpointTests.cs` covers scope enforcement and the build-list happy path using the in-memory store. Future work should add overlay coverage once Cartographer adapters are available. + +## Known gaps / TODO + +- Persist jobs to Scheduler storage and publish `scheduler.graph.job.completed@1` events + outbound webhook to Cartographer (see new `SCHED-WEB-21-004`). +- Extend `GET /graphs/jobs` with pagination cursors shared with Cartographer/Console. diff --git a/src/StellaOps.Scheduler.Worker.Tests/GlobalUsings.cs b/src/StellaOps.Scheduler.Worker.Tests/GlobalUsings.cs new file mode 100644 index 00000000..ed5b0ba1 --- /dev/null +++ b/src/StellaOps.Scheduler.Worker.Tests/GlobalUsings.cs @@ -0,0 +1,6 @@ +global using System.Collections.Immutable; +global using Moq; +global using StellaOps.Scheduler.ImpactIndex; +global using StellaOps.Scheduler.Models; +global using StellaOps.Scheduler.Worker; +global using Xunit; diff --git a/src/StellaOps.Scheduler.Worker.Tests/ImpactTargetingServiceTests.cs b/src/StellaOps.Scheduler.Worker.Tests/ImpactTargetingServiceTests.cs new file mode 100644 index 00000000..809614f8 --- /dev/null +++ b/src/StellaOps.Scheduler.Worker.Tests/ImpactTargetingServiceTests.cs @@ -0,0 +1,74 @@ +namespace StellaOps.Scheduler.Worker.Tests; + +public sealed class ImpactTargetingServiceTests +{ + [Fact] + public async Task ResolveByPurlsAsync_DeduplicatesKeysAndInvokesIndex() + { + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + var expected = CreateEmptyImpactSet(selector, usageOnly: false); + + var mockIndex = new Mock(MockBehavior.Strict); + mockIndex + .Setup(index => index.ResolveByPurlsAsync( + It.Is>(keys => keys.SequenceEqual(new[] { "pkg:npm/a", "pkg:npm/b" })), + false, + selector, + It.IsAny())) + .ReturnsAsync(expected); + + var service = new ImpactTargetingService(mockIndex.Object); + + var result = await service.ResolveByPurlsAsync( + new[] { "pkg:npm/a", "pkg:npm/A ", null!, "pkg:npm/b" }, + usageOnly: false, + selector); + + Assert.Same(expected, result); + mockIndex.VerifyAll(); + } + + [Fact] + public async Task ResolveByVulnerabilitiesAsync_ReturnsEmptyWhenNoIds() + { + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + var mockIndex = new Mock(MockBehavior.Strict); + var service = new ImpactTargetingService(mockIndex.Object); + + var result = await service.ResolveByVulnerabilitiesAsync(Array.Empty(), usageOnly: true, selector); + + Assert.Empty(result.Images); + Assert.True(result.UsageOnly); + mockIndex.Verify(index => index.ResolveByVulnerabilitiesAsync(It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task ResolveAllAsync_DelegatesToIndex() + { + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + var expected = CreateEmptyImpactSet(selector, usageOnly: true); + + var mockIndex = new Mock(); + mockIndex + .Setup(index => index.ResolveAllAsync(selector, true, It.IsAny())) + .Returns(new ValueTask(expected)); + + var service = new ImpactTargetingService(mockIndex.Object); + var result = await service.ResolveAllAsync(selector, usageOnly: true); + + Assert.Same(expected, result); + mockIndex.VerifyAll(); +} + + private static ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly) + { + return new ImpactSet( + selector, + ImmutableArray.Empty, + usageOnly, + DateTimeOffset.UtcNow, + 0, + snapshotId: null, + schemaVersion: SchedulerSchemaVersions.ImpactSet); + } +} diff --git a/src/StellaOps.Scheduler.Worker.Tests/StellaOps.Scheduler.Worker.Tests.csproj b/src/StellaOps.Scheduler.Worker.Tests/StellaOps.Scheduler.Worker.Tests.csproj new file mode 100644 index 00000000..f22c1ee5 --- /dev/null +++ b/src/StellaOps.Scheduler.Worker.Tests/StellaOps.Scheduler.Worker.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Scheduler.Worker/ImpactTargetingService.cs b/src/StellaOps.Scheduler.Worker/ImpactTargetingService.cs new file mode 100644 index 00000000..a3536a0a --- /dev/null +++ b/src/StellaOps.Scheduler.Worker/ImpactTargetingService.cs @@ -0,0 +1,104 @@ +using System.Collections.Immutable; +using StellaOps.Scheduler.ImpactIndex; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Worker; + +public interface IImpactTargetingService +{ + ValueTask ResolveByPurlsAsync( + IEnumerable productKeys, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default); + + ValueTask ResolveByVulnerabilitiesAsync( + IEnumerable vulnerabilityIds, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default); + + ValueTask ResolveAllAsync( + Selector selector, + bool usageOnly, + CancellationToken cancellationToken = default); +} + +public sealed class ImpactTargetingService : IImpactTargetingService +{ + private readonly IImpactIndex _impactIndex; + private readonly TimeProvider _timeProvider; + + public ImpactTargetingService(IImpactIndex impactIndex, TimeProvider? timeProvider = null) + { + _impactIndex = impactIndex ?? throw new ArgumentNullException(nameof(impactIndex)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask ResolveByPurlsAsync( + IEnumerable productKeys, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(productKeys); + ArgumentNullException.ThrowIfNull(selector); + + var distinct = productKeys + .Where(static key => !string.IsNullOrWhiteSpace(key)) + .Select(static key => key.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (distinct.Length == 0) + { + return CreateEmptyImpactSet(selector, usageOnly); + } + + return await _impactIndex.ResolveByPurlsAsync(distinct, usageOnly, selector, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask ResolveByVulnerabilitiesAsync( + IEnumerable vulnerabilityIds, + bool usageOnly, + Selector selector, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(vulnerabilityIds); + ArgumentNullException.ThrowIfNull(selector); + + var distinct = vulnerabilityIds + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (distinct.Length == 0) + { + return CreateEmptyImpactSet(selector, usageOnly); + } + + return await _impactIndex.ResolveByVulnerabilitiesAsync(distinct, usageOnly, selector, cancellationToken).ConfigureAwait(false); + } + + public ValueTask ResolveAllAsync( + Selector selector, + bool usageOnly, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(selector); + return _impactIndex.ResolveAllAsync(selector, usageOnly, cancellationToken); + } + + private ImpactSet CreateEmptyImpactSet(Selector selector, bool usageOnly) + { + return new ImpactSet( + selector, + ImmutableArray.Empty, + usageOnly, + _timeProvider.GetUtcNow(), + total: 0, + snapshotId: null, + schemaVersion: SchedulerSchemaVersions.ImpactSet); + } +} diff --git a/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj b/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj index c444aa16..fe04c962 100644 --- a/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj +++ b/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj @@ -1,8 +1,11 @@ - - - net10.0 - enable - enable - Exe - - + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Scheduler.Worker/TASKS.md b/src/StellaOps.Scheduler.Worker/TASKS.md index 262a9eaf..0f56220f 100644 --- a/src/StellaOps.Scheduler.Worker/TASKS.md +++ b/src/StellaOps.Scheduler.Worker/TASKS.md @@ -3,7 +3,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCHED-WORKER-16-201 | TODO | Scheduler Worker Guild | SCHED-QUEUE-16-401 | Planner loop (cron + event triggers) with lease management, fairness, and rate limiting (§6). | Planner integration tests cover cron/event triggers; rate limits enforced; logs include run IDs. | -| SCHED-WORKER-16-202 | TODO | Scheduler Worker Guild | SCHED-IMPACT-16-301 | Wire ImpactIndex targeting (ResolveByPurls/vulns), dedupe, shard planning. | Targeting tests confirm correct image selection; dedupe documented; shards evenly distributed. | +| SCHED-WORKER-16-202 | DOING (2025-10-26) | Scheduler Worker Guild | SCHED-IMPACT-16-301 | Wire ImpactIndex targeting (ResolveByPurls/vulns), dedupe, shard planning. | Targeting tests confirm correct image selection; dedupe documented; shards evenly distributed. | | SCHED-WORKER-16-203 | TODO | Scheduler Worker Guild | SCHED-WORKER-16-202 | Runner execution: call Scanner `/reports` (analysis-only) or `/scans` when configured; collect deltas; handle retries. | Runner tests stub Scanner; retries/backoff validated; deltas aggregated deterministically. | | SCHED-WORKER-16-204 | TODO | Scheduler Worker Guild | SCHED-WORKER-16-203 | Emit events (`scheduler.rescan.delta`, `scanner.report.ready`) for Notify/UI with summaries. | Events published to queue; payload schema documented; integration tests verify consumption. | | SCHED-WORKER-16-205 | TODO | Scheduler Worker Guild | SCHED-WORKER-16-201 | Metrics/telemetry: run stats, queue depth, planner latency, delta counts. | Metrics exported per spec; dashboards updated; alerts configured. | @@ -13,6 +13,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCHED-WORKER-20-301 | TODO | Scheduler Worker Guild, Policy Guild | SCHED-WORKER-16-201, POLICY-ENGINE-20-000 | Extend scheduler worker to trigger policy runs (full/incremental/simulate) via Policy Engine API, with idempotent job tracking and tenant scoping. | Worker schedules policy jobs deterministically; job records persisted; integration tests cover modes + cancellation. | +> 2025-10-26: Align worker serializers with `PolicyRunRequest/Status/DiffSummary` contracts from `src/StellaOps.Scheduler.Models`. Reference fixtures in `samples/api/scheduler/` for expected payload ordering. | SCHED-WORKER-20-302 | TODO | Scheduler Worker Guild | SCHED-WORKER-20-301, POLICY-ENGINE-20-006 | Implement policy delta targeting to re-evaluate only affected SBOM sets based on change streams and policy metadata. | Targeting reduces workload per design; tests simulate advisory/vex updates; metrics show delta counts. | | SCHED-WORKER-20-303 | TODO | Scheduler Worker Guild, Observability Guild | SCHED-WORKER-20-301 | Expose metrics (`policy_runs_scheduled`, `policy_runs_failed`, planner latency) and structured logs with policy/run identifiers. | Metrics registered; dashboards updated; logs validated in integration tests. | diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj index a02d954a..18e62f79 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj b/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj index d05901a6..7262701f 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj +++ b/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj @@ -1,26 +1,26 @@ - - - net10.0 - preview - enable - enable - true - false - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj b/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj index 63e1750a..b3d7759d 100644 --- a/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj @@ -1,29 +1,29 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.TaskRunner/StellaOps.TaskRunner.WebService/StellaOps.TaskRunner.WebService.csproj b/src/StellaOps.TaskRunner/StellaOps.TaskRunner.WebService/StellaOps.TaskRunner.WebService.csproj index 0fde3b90..f1ea48fa 100644 --- a/src/StellaOps.TaskRunner/StellaOps.TaskRunner.WebService/StellaOps.TaskRunner.WebService.csproj +++ b/src/StellaOps.TaskRunner/StellaOps.TaskRunner.WebService/StellaOps.TaskRunner.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/StellaOps.TaskRunner.Worker.csproj b/src/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/StellaOps.TaskRunner.Worker.csproj index 20b9cdad..db4d11b0 100644 --- a/src/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/StellaOps.TaskRunner.Worker.csproj +++ b/src/StellaOps.TaskRunner/StellaOps.TaskRunner.Worker/StellaOps.TaskRunner.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.WebService/StellaOps.TimelineIndexer.WebService.csproj b/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.WebService/StellaOps.TimelineIndexer.WebService.csproj index b3e79804..c250b20e 100644 --- a/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.WebService/StellaOps.TimelineIndexer.WebService.csproj +++ b/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.WebService/StellaOps.TimelineIndexer.WebService.csproj @@ -18,7 +18,7 @@ - + diff --git a/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/StellaOps.TimelineIndexer.Worker.csproj b/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/StellaOps.TimelineIndexer.Worker.csproj index 78834228..14ea452e 100644 --- a/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/StellaOps.TimelineIndexer.Worker.csproj +++ b/src/StellaOps.TimelineIndexer/StellaOps.TimelineIndexer.Worker/StellaOps.TimelineIndexer.Worker.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/StellaOps.UI/TASKS.md b/src/StellaOps.UI/TASKS.md index ebf64b24..c0110915 100644 --- a/src/StellaOps.UI/TASKS.md +++ b/src/StellaOps.UI/TASKS.md @@ -20,12 +20,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| -| UI-GRAPH-21-001 | TODO | UI Guild, Cartographer Guild | WEB-GRAPH-21-001, CARTO-GRAPH-21-006 | Implement virtualized pan/zoom canvas with clustering, severity/status overlays, and deterministic rendering budgets. | Canvas renders 50k-node fixtures <2s viewport; clustering verified; Cypress visual regression recorded. | -| UI-GRAPH-21-002 | TODO | UI Guild | UI-GRAPH-21-001, WEB-GRAPH-21-001 | Build inspector panel showing node metadata, effective findings, VEX rationale links, and dependent counts with copy/export support. | Inspector surfaces overlays per policy; links to explain pages work; unit/e2e tests cover scenarios. | -| UI-GRAPH-21-003 | TODO | UI Guild | UI-GRAPH-21-001, WEB-GRAPH-21-002 | Add filter/search experience (severity, status, scope, supplier, regex) with debounced API calls and virtualized results. | Filters update <500 ms p95 on fixtures; state reflected in permalink; tests cover combinations. | -| UI-GRAPH-21-004 | TODO | UI Guild, Policy Guild | UI-GRAPH-21-001, WEB-GRAPH-21-004, POLICY-ENGINE-30-002 | Implement path view (k-shortest paths, highlighting, export) and simulation overlay toggle. | Paths highlight edges, export JSON works, simulation overlay toggles without reload; e2e coverage added. | -| UI-GRAPH-21-005 | TODO | UI Guild | UI-GRAPH-21-001, CARTO-GRAPH-21-006 | Deliver time travel + diff view across SBOM versions with color coding for added/removed nodes/edges/overlays. | Diff view validated with fixtures; accessibility audit passes; screenshot baseline stored. | -| UI-GRAPH-21-006 | TODO | UI Guild, Accessibility Guild | UI-GRAPH-21-001 | Ship accessibility features (keyboard nav, high-contrast mode, colorblind palettes) and snapshot permalinks. | Keyboard navigation tested; WCAG checklist satisfied; permalinks reproduce state after reload. | +| UI-GRAPH-21-001 | TODO | UI Guild | WEB-GRAPH-21-001, AUTH-GRAPH-21-001 | Align Graph Explorer auth configuration with new `graph:*` scopes; consume scope identifiers from shared `StellaOpsScopes` exports (via generated SDK/config) instead of hard-coded strings. | UI requests graph tokens using shared scope constants; configuration docs updated; Cypress auth stub updated accordingly. | ## Link-Not-Merge v1 (Sprint 22) @@ -52,7 +47,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | UI-GRAPH-24-001 | TODO | UI Guild, SBOM Service Guild | WEB-GRAPH-24-001 | Build Graph Explorer canvas with layered/radial layouts, virtualization, zoom/pan, and scope toggles; initial render <1.5s for sample asset. | Canvas meets perf budget; automated tests cover navigation; accessibility validation done. | -| UI-GRAPH-24-002 | TODO | UI Guild, Policy Guild | UI-GRAPH-24-001, WEB-GRAPH-24-002 | Implement overlays (Policy, Evidence, License, Exposure) with tooltips, badges, and AOC indicators. | Overlays switch under 250ms; tooltips show explain links; tests cover overlay combos. | +| UI-GRAPH-24-002 | TODO | UI Guild, Policy Guild | UI-GRAPH-24-001, WEB-GRAPH-24-001, WEB-VEX-30-007 | Implement overlays (Policy, Evidence, License, Exposure), simulation toggle, path view, and SBOM diff/time-travel with accessible tooltips/AOC indicators. | Overlays + simulation toggle respond <250 ms; path view/diff export validated; accessibility tests cover keyboard + contrast; e2e covers overlay combos. | | UI-GRAPH-24-003 | TODO | UI Guild | UI-GRAPH-24-001 | Deliver filters/search panel with facets, saved views, permalinks, and share modal. | Filters update view <250ms; saved view persisted; permalinks reproduce state. | | UI-GRAPH-24-004 | TODO | UI Guild | UI-GRAPH-24-001 | Add side panels (Details, What-if, History) with upgrade simulation integration and SBOM diff viewer. | Simulation results display diff + policy impact; history shows added/removed nodes; tests cover flows. | | UI-GRAPH-24-006 | TODO | UI Guild, Accessibility Guild | UI-GRAPH-24-001..005 | Ensure accessibility (keyboard nav, screen reader labels, contrast), add hotkeys (`f`,`e`,`.`), and analytics instrumentation. | Accessibility audit passes; hotkeys documented; telemetry events captured. | diff --git a/src/StellaOps.VexLens/TASKS.md b/src/StellaOps.VexLens/TASKS.md index 78a70bbd..423ffb77 100644 --- a/src/StellaOps.VexLens/TASKS.md +++ b/src/StellaOps.VexLens/TASKS.md @@ -4,11 +4,11 @@ | VEXLENS-30-001 | TODO | VEX Lens Guild | EXCITITOR-LNM-21-001, CONCELIER-LNM-21-001 | Implement normalization pipeline for CSAF VEX, OpenVEX, CycloneDX VEX (status mapping, justification mapping, product tree parsing). | Normalization outputs deterministic canonical JSON; fixtures cover formats; unit tests pass. | | VEXLENS-30-002 | TODO | VEX Lens Guild | VEXLENS-30-001, SBOM-VULN-29-001 | Build product mapping library (CPE/CPE2.3/vendor tokens → purl/version) with scope quality scoring and path metadata. | Mapping library handles target ecosystems with property tests; scope scores recorded; docs updated. | | VEXLENS-30-003 | TODO | VEX Lens Guild, Issuer Directory Guild | ISSUER-30-001 | Integrate signature verification (Ed25519, DSSE, PKIX) using issuer keys, annotate evidence with verification state and failure reasons. | Signatures verified; failures logged; tests cover signed/unsigned/expired cases. | -| VEXLENS-30-004 | TODO | VEX Lens Guild, Policy Guild | POLICY-ENGINE-29-001 | Implement trust weighting engine (issuer base weights, signature modifiers, recency decay, justification modifiers, scope score adjustments) controlled by policy config. | Weighting functions configurable; policy overrides applied; unit tests validate formulas. | +| VEXLENS-30-004 | TODO | VEX Lens Guild, Policy Guild | POLICY-ENGINE-30-101 | Implement trust weighting engine (issuer base weights, signature modifiers, recency decay, justification modifiers, scope score adjustments) controlled by policy config. | Weighting functions configurable; policy overrides applied; unit tests validate formulas. | | VEXLENS-30-005 | TODO | VEX Lens Guild | VEXLENS-30-001..004 | Implement consensus algorithm producing `consensus_state`, `confidence`, `weights`, `quorum`, `rationale`; support states: NOT_AFFECTED, AFFECTED, FIXED, UNDER_INVESTIGATION, DISPUTED, INCONCLUSIVE. | Algorithm deterministic; unit/property tests cover conflict scenarios; rationale includes top evidences; docs drafted. | | VEXLENS-30-006 | TODO | VEX Lens Guild, Findings Ledger Guild | VEXLENS-30-005, LEDGER-29-003 | Materialize consensus projection storage with idempotent workers triggered by VEX/Policy changes; expose change events for downstream consumers. | Projection generated for fixtures; backpressure metrics recorded; replay harness passes. | | VEXLENS-30-007 | TODO | VEX Lens Guild | VEXLENS-30-006 | Expose APIs (`/vex/consensus`, `/vex/consensus/query`, `/vex/consensus/{id}`, `/vex/consensus/simulate`, `/vex/consensus/export`) with pagination, cost budgets, and OpenAPI docs. | APIs deployed with schema validation; integration tests cover filters/simulation/export; rate limits enforced. | -| VEXLENS-30-008 | TODO | VEX Lens Guild, Policy Guild | VEXLENS-30-006, POLICY-ENGINE-29-001 | Integrate consensus signals with Policy Engine (thresholds, suppression, simulation inputs) and Vuln Explorer detail view. | Policy consumes consensus via documented contract; Vuln Explorer shows consensus chip; e2e tests confirm suppression behavior. | +| VEXLENS-30-008 | TODO | VEX Lens Guild, Policy Guild | VEXLENS-30-006, POLICY-ENGINE-30-101 | Integrate consensus signals with Policy Engine (thresholds, suppression, simulation inputs) and Vuln Explorer detail view. | Policy consumes consensus via documented contract; Vuln Explorer shows consensus chip; e2e tests confirm suppression behavior. | | VEXLENS-30-009 | TODO | VEX Lens Guild, Observability Guild | VEXLENS-30-006..008 | Instrument metrics (`vex_consensus_compute_latency`, `vex_consensus_disputed_total`, `vex_signature_verification_rate`), structured logs, and traces; publish dashboards/alerts. | Metrics/traces live; dashboards approved; alert thresholds configured. | | VEXLENS-30-010 | TODO | VEX Lens Guild, QA Guild | VEXLENS-30-001..008 | Develop unit/property/integration/load tests (10M records), determinism harness, fuzz testing for malformed product trees. | Test suites green; load tests documented; determinism harness validated across two runs. | | VEXLENS-30-011 | TODO | VEX Lens Guild, DevOps Guild | VEXLENS-30-006..009 | Provide deployment manifests, caching configuration, scaling guides, offline kit seeds, and runbooks. | Deployment docs merged; smoke deploy validated; offline kit updated; runbooks published. | diff --git a/src/StellaOps.Web/TASKS.md b/src/StellaOps.Web/TASKS.md index 5dcbd25d..e08caf23 100644 --- a/src/StellaOps.Web/TASKS.md +++ b/src/StellaOps.Web/TASKS.md @@ -1,9 +1,11 @@ # TASKS — Epic 1: Aggregation-Only Contract | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| WEB-AOC-19-001 `Shared AOC guard primitives` | TODO | BE-Base Platform Guild | — | Provide `AOCForbiddenKeys`, guard middleware/interceptor hooks, and error types (`AOCError`, `AOCViolationCode`) for ingestion services. Publish sample usage + analyzer to ensure guard registered. | +| WEB-AOC-19-001 `Shared AOC guard primitives` | DOING (2025-10-26) | BE-Base Platform Guild | — | Provide `AOCForbiddenKeys`, guard middleware/interceptor hooks, and error types (`AOCError`, `AOCViolationCode`) for ingestion services. Publish sample usage + analyzer to ensure guard registered. | +> 2025-10-26: Introduced `StellaOps.Aoc` library with forbidden key list, guard result/options, and baseline write guard + tests. Middleware/analyzer wiring still pending. | WEB-AOC-19-002 `Provenance & signature helpers` | TODO | BE-Base Platform Guild | WEB-AOC-19-001 | Ship `ProvenanceBuilder`, checksum utilities, and signature verification helper integrated with guard logging. Cover DSSE/CMS formats with unit tests. | | WEB-AOC-19-003 `Analyzer + test fixtures` | TODO | QA Guild, BE-Base Platform Guild | WEB-AOC-19-001 | Author Roslyn analyzer preventing ingestion modules from writing forbidden keys without guard, and provide shared test fixtures for guard validation used by Concelier/Excititor service tests. | +> Docs alignment (2025-10-26): Analyzer expectations detailed in `docs/ingestion/aggregation-only-contract.md` §3/5; CI integration tracked via DEVOPS-AOC-19-001. ## Policy Engine v2 @@ -18,18 +20,17 @@ | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| WEB-GRAPH-21-001 `Graph endpoints` | TODO | BE-Base Platform Guild, Cartographer Guild | CARTO-GRAPH-21-006, AUTH-GRAPH-21-001 | Add gateway routes for graph versions/viewport/node/path/diff/export/simulate endpoints with tenant enforcement, scope checks, and streaming responses. | +| WEB-GRAPH-21-001 `Graph endpoints` | TODO | BE-Base Platform Guild, Graph Platform Guild | GRAPH-API-28-003, AUTH-VULN-24-001 | Add gateway routes for graph versions/viewport/node/path/diff/export endpoints with tenant enforcement, scope checks, and streaming responses; proxy Policy Engine diff toggles without inline logic. Adopt `StellaOpsScopes` constants for RBAC enforcement. | | WEB-GRAPH-21-002 `Request validation` | TODO | BE-Base Platform Guild | WEB-GRAPH-21-001 | Implement bbox/zoom/path parameter validation, pagination tokens, and deterministic ordering; add contract tests for boundary conditions. | | WEB-GRAPH-21-003 `Error mapping & exports` | TODO | BE-Base Platform Guild, QA Guild | WEB-GRAPH-21-001 | Map graph service errors to `ERR_Graph_*`, support GraphML/JSONL export streaming, and document rate limits. | -| WEB-GRAPH-21-004 `Simulation bridge` | TODO | BE-Base Platform Guild, Policy Guild | WEB-GRAPH-21-001, POLICY-ENGINE-30-002 | Wire Policy Engine simulation overlays into graph endpoints (`simulate`, `filter`, `paths`) without persisting state; ensure latency budgets met. | +| WEB-GRAPH-21-004 `Overlay pass-through` | TODO | BE-Base Platform Guild, Policy Guild | WEB-GRAPH-21-001, POLICY-ENGINE-30-002 | Proxy Policy Engine overlay responses for graph endpoints while keeping gateway stateless; maintain streaming budgets and latency SLOs. | ## Graph Explorer (Sprint 28) | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| WEB-GRAPH-24-001 `Gateway routing refresh` | TODO | BE-Base Platform Guild | GRAPH-API-28-001, AUTH-GRAPH-21-001 | Route `/graph/*` requests to new Graph API service, inject tenant headers, feature flags, and enforce scope checks (`graph:read`, `graph:query`, `graph:export`). | -| WEB-GRAPH-24-002 `Overlay proxy` | TODO | BE-Base Platform Guild, Policy Guild | WEB-GRAPH-24-001, GRAPH-API-28-006 | Proxy policy/advisory/VEX overlay responses from dedicated services, manage caching headers/invalidation, and ensure no simulation logic exists in gateway. | -| WEB-GRAPH-24-004 `Telemetry aggregation` | TODO | BE-Base Platform Guild, Observability Guild | WEB-GRAPH-24-001..002, DEVOPS-GRAPH-28-003 | Collect gateway metrics/logs (tile latency, proxy errors, overlay cache stats) and forward to dashboards; document sampling strategy. | +| WEB-GRAPH-24-001 `Gateway proxy refresh` | TODO | BE-Base Platform Guild | GRAPH-API-28-001, AUTH-GRAPH-21-001 | Gateway proxy for Graph API and Policy overlays with RBAC, caching, pagination, ETags, and streaming; zero business logic. | +| WEB-GRAPH-24-004 `Telemetry aggregation` | TODO | BE-Base Platform Guild, Observability Guild | WEB-GRAPH-24-001, DEVOPS-GRAPH-28-003 | Collect gateway metrics/logs (tile latency, proxy errors, overlay cache stats) and forward to dashboards; document sampling strategy. | ## Link-Not-Merge v1 @@ -52,9 +53,8 @@ | ID | Status | Owner(s) | Depends on | Notes | |----|--------|----------|------------|-------| -| WEB-GRAPH-24-001 `Graph endpoints` | TODO | BE-Base Platform Guild, SBOM Service Guild | SBOM-GRAPH-24-002 | Implement `/graph/assets/*` endpoints (snapshots, adjacency, search) with pagination, ETags, and tenant scoping. | -| WEB-GRAPH-24-002 `Simulation proxy` | TODO | BE-Base Platform Guild, Policy Guild | WEB-GRAPH-24-001, POLICY-ENGINE-50-002 | Proxy upgrade/policy simulation calls to Policy Engine, enforce rate limits, and return streamed diffs without gateway-side computation. | -| WEB-GRAPH-24-004 `AOC enrichers` | TODO | BE-Base Platform Guild | WEB-GRAPH-24-001, WEB-GRAPH-24-002 | Embed AOC summaries sourced from overlay services; ensure gateway does not compute derived severity or hints. | +| WEB-GRAPH-24-001 `Graph endpoints` | TODO | BE-Base Platform Guild, SBOM Service Guild | SBOM-GRAPH-24-002 | Implement `/graph/assets/*` endpoints (snapshots, adjacency, search) with pagination, ETags, and tenant scoping while acting as a pure proxy. | +| WEB-GRAPH-24-004 `AOC enrichers` | TODO | BE-Base Platform Guild | WEB-GRAPH-24-001 | Embed AOC summaries sourced from overlay services; ensure gateway does not compute derived severity or hints. | ## StellaOps Console (Sprint 23) @@ -100,6 +100,7 @@ | WEB-VULN-29-002 `Ledger proxy headers` | TODO | BE-Base Platform Guild, Findings Ledger Guild | WEB-VULN-29-001, LEDGER-29-002 | Forward workflow actions to Findings Ledger with idempotency headers and correlation IDs; handle retries/backoff. | | WEB-VULN-29-003 `Simulation + export routing` | TODO | BE-Base Platform Guild | VULN-API-29-005, VULN-API-29-008 | Provide simulation and export orchestration routes with SSE/progress headers, signed download links, and request budgeting. | | WEB-VULN-29-004 `Telemetry aggregation` | TODO | BE-Base Platform Guild, Observability Guild | WEB-VULN-29-001..003, DEVOPS-VULN-29-003 | Emit gateway metrics/logs (latency, error rates, export duration), propagate query hashes for analytics dashboards. | +| WEB-VEX-30-007 `VEX consensus routing` | TODO | BE-Base Platform Guild, VEX Lens Guild | VEXLENS-30-007, AUTH-VULN-24-001 | Route `/vex/consensus` APIs with tenant RBAC/ABAC, caching, and streaming; surface telemetry and trace IDs without gateway-side overlay logic. | ## Advisory AI (Sprint 31) diff --git a/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj b/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj index 783ae2d3..427ec8c2 100644 --- a/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj +++ b/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj b/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj index 2294fcc2..cbd64140 100644 --- a/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj +++ b/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - false - true - false - - - - - - - - - - + + + net10.0 + preview + enable + enable + false + true + false + + + + + + + + + + diff --git a/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj b/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj index 4e4cc932..1c7c4131 100644 --- a/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj +++ b/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj @@ -1,15 +1,15 @@ - - - net10.0 - preview - enable - enable - true - StellaOps.Zastava.Webhook - $(NoWarn);CA2254 - + + + net10.0 + preview + enable + enable + true + StellaOps.Zastava.Webhook + $(NoWarn);CA2254 + - + diff --git a/src/StellaOps.sln b/src/StellaOps.sln index 8d7a1025..3756ce0a 100644 --- a/src/StellaOps.sln +++ b/src/StellaOps.sln @@ -345,6 +345,28 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cartographer", "S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.SbomService", "StellaOps.SbomService\StellaOps.SbomService.csproj", "{3510DF3E-E822-4FB1-8C65-ED6DBAD223D4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.WebService", "StellaOps.Scheduler.WebService\StellaOps.Scheduler.WebService.csproj", "{C733F161-FCED-4D21-BC83-5CC079E93547}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scheduler.WebService.Tests", "StellaOps.Scheduler.WebService.Tests\StellaOps.Scheduler.WebService.Tests.csproj", "{76E1E74F-41C1-4E24-85EA-ED13F28B80B1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Registry.TokenService", "StellaOps.Registry.TokenService\StellaOps.Registry.TokenService.csproj", "{EC73D558-0472-49E2-B46E-D26F9686AA9C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Registry.TokenService.Tests", "StellaOps.Registry.TokenService.Tests\StellaOps.Registry.TokenService.Tests.csproj", "{1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "StellaOps.Bench", "StellaOps.Bench", "{1553F566-661E-A2F5-811B-F74BF45C44CC}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PolicyEngine", "PolicyEngine", "{CBDF819E-923F-A07F-78D9-D599DD28197E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Bench.PolicyEngine", "StellaOps.Bench\PolicyEngine\StellaOps.Bench.PolicyEngine\StellaOps.Bench.PolicyEngine.csproj", "{D8B22C17-28E9-4059-97C5-4AC4600A2BD5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Aoc", "StellaOps.Aoc\StellaOps.Aoc.csproj", "{6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Aoc.Tests", "StellaOps.Aoc.Tests\StellaOps.Aoc.Tests.csproj", "{4D167781-1AC0-46CF-A32E-1B6E048940B2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.RawModels", "StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj", "{C3AEAEE7-038E-45FF-892B-DB18EE29F790}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.RawModels.Tests", "StellaOps.Concelier.RawModels.Tests\StellaOps.Concelier.RawModels.Tests.csproj", "{7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -2347,6 +2369,114 @@ Global {3510DF3E-E822-4FB1-8C65-ED6DBAD223D4}.Release|x64.Build.0 = Release|Any CPU {3510DF3E-E822-4FB1-8C65-ED6DBAD223D4}.Release|x86.ActiveCfg = Release|Any CPU {3510DF3E-E822-4FB1-8C65-ED6DBAD223D4}.Release|x86.Build.0 = Release|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Debug|x64.ActiveCfg = Debug|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Debug|x64.Build.0 = Debug|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Debug|x86.ActiveCfg = Debug|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Debug|x86.Build.0 = Debug|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Release|Any CPU.Build.0 = Release|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Release|x64.ActiveCfg = Release|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Release|x64.Build.0 = Release|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Release|x86.ActiveCfg = Release|Any CPU + {C733F161-FCED-4D21-BC83-5CC079E93547}.Release|x86.Build.0 = Release|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Debug|x64.ActiveCfg = Debug|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Debug|x64.Build.0 = Debug|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Debug|x86.ActiveCfg = Debug|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Debug|x86.Build.0 = Debug|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Release|Any CPU.Build.0 = Release|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Release|x64.ActiveCfg = Release|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Release|x64.Build.0 = Release|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Release|x86.ActiveCfg = Release|Any CPU + {76E1E74F-41C1-4E24-85EA-ED13F28B80B1}.Release|x86.Build.0 = Release|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Debug|x64.ActiveCfg = Debug|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Debug|x64.Build.0 = Debug|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Debug|x86.ActiveCfg = Debug|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Debug|x86.Build.0 = Debug|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Release|Any CPU.Build.0 = Release|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Release|x64.ActiveCfg = Release|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Release|x64.Build.0 = Release|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Release|x86.ActiveCfg = Release|Any CPU + {EC73D558-0472-49E2-B46E-D26F9686AA9C}.Release|x86.Build.0 = Release|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Debug|x64.ActiveCfg = Debug|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Debug|x64.Build.0 = Debug|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Debug|x86.ActiveCfg = Debug|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Debug|x86.Build.0 = Debug|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Release|Any CPU.Build.0 = Release|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Release|x64.ActiveCfg = Release|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Release|x64.Build.0 = Release|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Release|x86.ActiveCfg = Release|Any CPU + {1E532EAB-8DB7-42DF-A9BD-BBBA08C8148F}.Release|x86.Build.0 = Release|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Debug|x64.ActiveCfg = Debug|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Debug|x64.Build.0 = Debug|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Debug|x86.ActiveCfg = Debug|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Debug|x86.Build.0 = Debug|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Release|Any CPU.Build.0 = Release|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Release|x64.ActiveCfg = Release|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Release|x64.Build.0 = Release|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Release|x86.ActiveCfg = Release|Any CPU + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5}.Release|x86.Build.0 = Release|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Debug|x64.ActiveCfg = Debug|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Debug|x64.Build.0 = Debug|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Debug|x86.ActiveCfg = Debug|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Debug|x86.Build.0 = Debug|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Release|Any CPU.Build.0 = Release|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Release|x64.ActiveCfg = Release|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Release|x64.Build.0 = Release|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Release|x86.ActiveCfg = Release|Any CPU + {6BE16682-4FB9-49C7-A2B3-ECB4EC5EF8BD}.Release|x86.Build.0 = Release|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Debug|x64.ActiveCfg = Debug|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Debug|x64.Build.0 = Debug|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Debug|x86.ActiveCfg = Debug|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Debug|x86.Build.0 = Debug|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Release|Any CPU.Build.0 = Release|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Release|x64.ActiveCfg = Release|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Release|x64.Build.0 = Release|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Release|x86.ActiveCfg = Release|Any CPU + {4D167781-1AC0-46CF-A32E-1B6E048940B2}.Release|x86.Build.0 = Release|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Debug|x64.Build.0 = Debug|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Debug|x86.ActiveCfg = Debug|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Debug|x86.Build.0 = Debug|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Release|Any CPU.Build.0 = Release|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Release|x64.ActiveCfg = Release|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Release|x64.Build.0 = Release|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Release|x86.ActiveCfg = Release|Any CPU + {C3AEAEE7-038E-45FF-892B-DB18EE29F790}.Release|x86.Build.0 = Release|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Debug|x64.Build.0 = Debug|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Debug|x86.Build.0 = Debug|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Release|Any CPU.Build.0 = Release|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Release|x64.ActiveCfg = Release|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Release|x64.Build.0 = Release|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Release|x86.ActiveCfg = Release|Any CPU + {7FACF6B4-7E12-4543-AAD4-0072FA1ECE0E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2454,5 +2584,7 @@ Global {F1F029E6-2E4B-4A42-8D8F-AB325EE3B608} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {CBE6E3D8-230C-4513-B98F-99D82B83B9F7} = {F1F029E6-2E4B-4A42-8D8F-AB325EE3B608} {821C7F88-B775-4D3C-8D89-850B6C34E818} = {F1F029E6-2E4B-4A42-8D8F-AB325EE3B608} + {CBDF819E-923F-A07F-78D9-D599DD28197E} = {1553F566-661E-A2F5-811B-F74BF45C44CC} + {D8B22C17-28E9-4059-97C5-4AC4600A2BD5} = {CBDF819E-923F-A07F-78D9-D599DD28197E} EndGlobalSection EndGlobal diff --git a/test/contract/AGENTS.md b/test/contract/AGENTS.md deleted file mode 100644 index 301ca4bd..00000000 --- a/test/contract/AGENTS.md +++ /dev/null @@ -1,15 +0,0 @@ -# Contract Testing Guild Charter - -## Mission -Guarantee runtime adherence to OpenAPI specifications through mock servers, replay suites, and golden example validation. - -## Scope -- Generate mock servers from aggregate OAS and maintain operation examples. -- Run replay tests against staging environments to detect schema drift. -- Manage golden fixtures used for SDK examples and documentation. -- Provide tooling for PR-level contract verification. - -## Definition of Done -- Mock server and replay suites run in CI gating merges. -- Golden examples kept in sync with docs and SDK snippets. -- Regression catches contract drift before release. diff --git a/test/contract/TASKS.md b/test/contract/TASKS.md deleted file mode 100644 index c7fbc857..00000000 --- a/test/contract/TASKS.md +++ /dev/null @@ -1,13 +0,0 @@ -# Contract Testing Task Board — Epic 17: SDKs & OpenAPI Docs - -## Sprint 62 – Mock Server & Fixtures -| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | -|----|--------|----------|------------|-------------|---------------| -| CONTR-62-001 | TODO | Contract Testing Guild | OAS-61-002 | Generate mock server configuration from aggregate spec; seed golden request/response fixtures. | Mock server runs locally; fixtures stored under version control; integration docs updated. | -| CONTR-62-002 | TODO | Contract Testing Guild | CONTR-62-001 | Integrate mock server tests into PR CI ensuring each operation has example coverage. | CI job added; failing on missing examples; results visible in PR checks. | - -## Sprint 63 – Replay & Drift Detection -| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | -|----|--------|----------|------------|-------------|---------------| -| CONTR-63-001 | TODO | Contract Testing Guild, Platform Guild | CONTR-62-002 | Build replay harness capturing staging traffic and validating against spec; enforce zero drift. | Replay suite runs nightly; failures create alerts; documentation updated. | -| CONTR-63-002 | TODO | Contract Testing Guild, Observability Guild | CONTR-63-001 | Emit metrics/logs for contract test coverage and failures. | Metrics visible in dashboards; alerts configured. | diff --git a/tmp/reflect/Program.cs b/tmp/reflect/Program.cs new file mode 100644 index 00000000..ed089f4c --- /dev/null +++ b/tmp/reflect/Program.cs @@ -0,0 +1,13 @@ +using System; +using System.Linq; +using System.Reflection; + +var assembly = Assembly.Load("RoaringBitmap"); +foreach (var type in assembly.GetTypes().OrderBy(t => t.FullName)) +{ + Console.WriteLine(type.FullName); + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) + { + Console.WriteLine($" {method.ReturnType.Name} {method.Name}({string.Join(", ", method.GetParameters().Select(p => p.ParameterType.Name + " " + p.Name))})"); + } +} diff --git a/tmp/reflect/reflect.csproj b/tmp/reflect/reflect.csproj new file mode 100644 index 00000000..9d15235d --- /dev/null +++ b/tmp/reflect/reflect.csproj @@ -0,0 +1,14 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj b/tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj new file mode 100644 index 00000000..5828535c --- /dev/null +++ b/tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj @@ -0,0 +1,18 @@ + + + + Exe + net10.0 + enable + enable + true + + + + + + + + + + diff --git a/tools/LanguageAnalyzerSmoke/Program.cs b/tools/LanguageAnalyzerSmoke/Program.cs new file mode 100644 index 00000000..1679f839 --- /dev/null +++ b/tools/LanguageAnalyzerSmoke/Program.cs @@ -0,0 +1,348 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scanner.Analyzers.Lang; +using StellaOps.Scanner.Analyzers.Lang.Plugin; +using StellaOps.Scanner.Core.Security; + +internal sealed record SmokeScenario(string Name, string[] UsageHintRelatives) +{ + public IReadOnlyList ResolveUsageHints(string scenarioRoot) + => UsageHintRelatives.Select(relative => Path.GetFullPath(Path.Combine(scenarioRoot, relative))).ToArray(); +} + +internal sealed class SmokeOptions +{ + public string RepoRoot { get; set; } = Directory.GetCurrentDirectory(); + public string PluginDirectoryName { get; set; } = "StellaOps.Scanner.Analyzers.Lang.Python"; + public string FixtureRelativePath { get; set; } = Path.Combine("src", "StellaOps.Scanner.Analyzers.Lang.Python.Tests", "Fixtures", "lang", "python"); + + public static SmokeOptions Parse(string[] args) + { + var options = new SmokeOptions(); + + for (var index = 0; index < args.Length; index++) + { + var current = args[index]; + switch (current) + { + case "--repo-root": + case "-r": + options.RepoRoot = RequireValue(args, ref index, current); + break; + case "--plugin-directory": + case "-p": + options.PluginDirectoryName = RequireValue(args, ref index, current); + break; + case "--fixture-path": + case "-f": + options.FixtureRelativePath = RequireValue(args, ref index, current); + break; + case "--help": + case "-h": + PrintUsage(); + Environment.Exit(0); + break; + default: + throw new ArgumentException($"Unknown argument '{current}'. Use --help for usage."); + } + } + + options.RepoRoot = Path.GetFullPath(options.RepoRoot); + return options; + } + + private static string RequireValue(string[] args, ref int index, string switchName) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException($"Missing value for '{switchName}'."); + } + + index++; + var value = args[index]; + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException($"Value for '{switchName}' cannot be empty."); + } + + return value; + } + + private static void PrintUsage() + { + Console.WriteLine("Language Analyzer Smoke Harness"); + Console.WriteLine("Usage: dotnet run --project tools/LanguageAnalyzerSmoke -- [options]"); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" -r, --repo-root Repository root (defaults to current working directory)"); + Console.WriteLine(" -p, --plugin-directory Analyzer plug-in directory under plugins/scanner/analyzers/lang (defaults to StellaOps.Scanner.Analyzers.Lang.Python)"); + Console.WriteLine(" -f, --fixture-path Relative path to fixtures root (defaults to src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python)"); + Console.WriteLine(" -h, --help Show usage information"); + } +} + +internal sealed record PluginManifest +{ + [JsonPropertyName("schemaVersion")] + public string SchemaVersion { get; init; } = string.Empty; + + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("displayName")] + public string DisplayName { get; init; } = string.Empty; + + [JsonPropertyName("version")] + public string Version { get; init; } = string.Empty; + + [JsonPropertyName("requiresRestart")] + public bool RequiresRestart { get; init; } + + [JsonPropertyName("entryPoint")] + public PluginEntryPoint EntryPoint { get; init; } = new(); + + [JsonPropertyName("capabilities")] + public IReadOnlyList Capabilities { get; init; } = Array.Empty(); + + [JsonPropertyName("metadata")] + public IReadOnlyDictionary Metadata { get; init; } = ImmutableDictionary.Empty; +} + +internal sealed record PluginEntryPoint +{ + [JsonPropertyName("type")] + public string Type { get; init; } = string.Empty; + + [JsonPropertyName("assembly")] + public string Assembly { get; init; } = string.Empty; + + [JsonPropertyName("typeName")] + public string TypeName { get; init; } = string.Empty; +} + +file static class Program +{ + private static readonly SmokeScenario[] PythonScenarios = + { + new("simple-venv", new[] { Path.Combine("bin", "simple-tool") }), + new("pip-cache", new[] { Path.Combine("lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool") }), + new("layered-editable", new[] { Path.Combine("layer1", "usr", "bin", "layered-cli") }) + }; + + public static async Task Main(string[] args) + { + try + { + var options = SmokeOptions.Parse(args); + await RunAsync(options).ConfigureAwait(false); + Console.WriteLine("✅ Python analyzer smoke checks passed"); + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"❌ {ex.Message}"); + return 1; + } + } + + private static async Task RunAsync(SmokeOptions options) + { + ValidateOptions(options); + + var pluginRoot = Path.Combine(options.RepoRoot, "plugins", "scanner", "analyzers", "lang", options.PluginDirectoryName); + var manifestPath = Path.Combine(pluginRoot, "manifest.json"); + if (!File.Exists(manifestPath)) + { + throw new FileNotFoundException($"Plug-in manifest not found at '{manifestPath}'.", manifestPath); + } + + using var manifestStream = File.OpenRead(manifestPath); + var manifest = JsonSerializer.Deserialize(manifestStream, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip + }) ?? throw new InvalidOperationException($"Unable to parse manifest '{manifestPath}'."); + + ValidateManifest(manifest, options.PluginDirectoryName); + + var pluginAssemblyPath = Path.Combine(pluginRoot, manifest.EntryPoint.Assembly); + if (!File.Exists(pluginAssemblyPath)) + { + throw new FileNotFoundException($"Plug-in assembly '{manifest.EntryPoint.Assembly}' not found under '{pluginRoot}'.", pluginAssemblyPath); + } + + var sha256 = ComputeSha256(pluginAssemblyPath); + Console.WriteLine($"→ Plug-in assembly SHA-256: {sha256}"); + + using var serviceProvider = BuildServiceProvider(); + var catalog = new LanguageAnalyzerPluginCatalog(new RestartOnlyPluginGuard(), NullLogger.Instance); + catalog.LoadFromDirectory(pluginRoot, seal: true); + + if (catalog.Plugins.Count == 0) + { + throw new InvalidOperationException($"No analyzer plug-ins were loaded from '{pluginRoot}'."); + } + + var analyzerSet = catalog.CreateAnalyzers(serviceProvider); + if (analyzerSet.Count == 0) + { + throw new InvalidOperationException("Language analyzer plug-ins reported no analyzers."); + } + + var analyzerIds = analyzerSet.Select(analyzer => analyzer.Id).ToArray(); + Console.WriteLine($"→ Loaded analyzers: {string.Join(", ", analyzerIds)}"); + + if (!analyzerIds.Contains("python", StringComparer.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Python analyzer was not created by the plug-in."); + } + + var fixtureRoot = Path.GetFullPath(Path.Combine(options.RepoRoot, options.FixtureRelativePath)); + if (!Directory.Exists(fixtureRoot)) + { + throw new DirectoryNotFoundException($"Fixture directory '{fixtureRoot}' does not exist."); + } + + foreach (var scenario in PythonScenarios) + { + await RunScenarioAsync(scenario, fixtureRoot, catalog, serviceProvider).ConfigureAwait(false); + } + } + + private static ServiceProvider BuildServiceProvider() + { + var services = new ServiceCollection(); + services.AddLogging(); + return services.BuildServiceProvider(); + } + + private static async Task RunScenarioAsync(SmokeScenario scenario, string fixtureRoot, ILanguageAnalyzerPluginCatalog catalog, IServiceProvider services) + { + var scenarioRoot = Path.Combine(fixtureRoot, scenario.Name); + if (!Directory.Exists(scenarioRoot)) + { + throw new DirectoryNotFoundException($"Scenario '{scenario.Name}' directory missing at '{scenarioRoot}'."); + } + + var goldenPath = Path.Combine(scenarioRoot, "expected.json"); + string? goldenNormalized = null; + if (File.Exists(goldenPath)) + { + goldenNormalized = NormalizeJson(await File.ReadAllTextAsync(goldenPath).ConfigureAwait(false)); + } + + var usageHints = new LanguageUsageHints(scenario.ResolveUsageHints(scenarioRoot)); + var context = new LanguageAnalyzerContext(scenarioRoot, TimeProvider.System, usageHints, services); + + var coldEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services)); + var coldStopwatch = Stopwatch.StartNew(); + var coldResult = await coldEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false); + coldStopwatch.Stop(); + + if (coldResult.Components.Count == 0) + { + throw new InvalidOperationException($"Scenario '{scenario.Name}' produced no components during cold run."); + } + + var coldJson = NormalizeJson(coldResult.ToJson(indent: true)); + if (goldenNormalized is string expected && !string.Equals(coldJson, expected, StringComparison.Ordinal)) + { + Console.WriteLine($"⚠️ Scenario '{scenario.Name}' output deviates from repository golden snapshot."); + } + + var warmEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services)); + var warmStopwatch = Stopwatch.StartNew(); + var warmResult = await warmEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false); + warmStopwatch.Stop(); + + var warmJson = NormalizeJson(warmResult.ToJson(indent: true)); + if (!string.Equals(coldJson, warmJson, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Scenario '{scenario.Name}' produced different outputs between cold and warm runs."); + } + + EnsureDurationWithinBudget(scenario.Name, coldStopwatch.Elapsed, warmStopwatch.Elapsed); + + Console.WriteLine($"✓ Scenario '{scenario.Name}' — components {coldResult.Components.Count}, cold {coldStopwatch.Elapsed.TotalMilliseconds:F1} ms, warm {warmStopwatch.Elapsed.TotalMilliseconds:F1} ms"); + } + + private static void EnsureDurationWithinBudget(string scenarioName, TimeSpan coldDuration, TimeSpan warmDuration) + { + var coldBudget = TimeSpan.FromSeconds(30); + var warmBudget = TimeSpan.FromSeconds(5); + + if (coldDuration > coldBudget) + { + throw new InvalidOperationException($"Scenario '{scenarioName}' cold run exceeded budget ({coldDuration.TotalSeconds:F2}s > {coldBudget.TotalSeconds:F2}s)."); + } + + if (warmDuration > warmBudget) + { + throw new InvalidOperationException($"Scenario '{scenarioName}' warm run exceeded budget ({warmDuration.TotalSeconds:F2}s > {warmBudget.TotalSeconds:F2}s)."); + } + } + + private static string NormalizeJson(string json) + => json.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); + + private static void ValidateOptions(SmokeOptions options) + { + if (!Directory.Exists(options.RepoRoot)) + { + throw new DirectoryNotFoundException($"Repository root '{options.RepoRoot}' does not exist."); + } + } + + private static void ValidateManifest(PluginManifest manifest, string expectedDirectory) + { + if (!string.Equals(manifest.SchemaVersion, "1.0", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Unexpected manifest schema version '{manifest.SchemaVersion}'."); + } + + if (!manifest.RequiresRestart) + { + throw new InvalidOperationException("Language analyzer plug-in must be marked as restart-only."); + } + + if (!string.Equals(manifest.EntryPoint.Type, "dotnet", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Unsupported entry point type '{manifest.EntryPoint.Type}'."); + } + + if (!manifest.Capabilities.Contains("python", StringComparer.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Manifest capabilities do not include 'python'."); + } + + if (!string.Equals(manifest.EntryPoint.TypeName, "StellaOps.Scanner.Analyzers.Lang.Python.PythonAnalyzerPlugin", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Unexpected entry point type name '{manifest.EntryPoint.TypeName}'."); + } + + if (!string.Equals(manifest.Id, "stellaops.analyzer.lang.python", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Manifest id '{manifest.Id}' does not match expected plug-in id for directory '{expectedDirectory}'."); + } + } + + private static string ComputeSha256(string path) + { + using var hash = SHA256.Create(); + using var stream = File.OpenRead(path); + var digest = hash.ComputeHash(stream); + var builder = new StringBuilder(digest.Length * 2); + foreach (var b in digest) + { + builder.Append(b.ToString("x2")); + } + return builder.ToString(); + } +} diff --git a/tools/PolicyDslValidator/PolicyDslValidator.csproj b/tools/PolicyDslValidator/PolicyDslValidator.csproj new file mode 100644 index 00000000..edd23828 --- /dev/null +++ b/tools/PolicyDslValidator/PolicyDslValidator.csproj @@ -0,0 +1,14 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/tools/PolicyDslValidator/Program.cs b/tools/PolicyDslValidator/Program.cs new file mode 100644 index 00000000..a1420e6e --- /dev/null +++ b/tools/PolicyDslValidator/Program.cs @@ -0,0 +1,56 @@ +using StellaOps.Policy; + +if (args.Length == 0) +{ + Console.Error.WriteLine("Usage: policy-dsl-validator [--strict] [--json] [ ...]"); + Console.Error.WriteLine("Example: policy-dsl-validator --strict docs/examples/policies"); + return 64; // EX_USAGE +} + +var inputs = new List(); +var strict = false; +var outputJson = false; + +foreach (var arg in args) +{ + switch (arg) + { + case "--strict": + case "-s": + strict = true; + break; + + case "--json": + case "-j": + outputJson = true; + break; + + case "--help": + case "-h": + case "-?": + Console.WriteLine("Usage: policy-dsl-validator [--strict] [--json] [ ...]"); + Console.WriteLine("Example: policy-dsl-validator --strict docs/examples/policies"); + return 0; + + default: + inputs.Add(arg); + break; + } +} + +if (inputs.Count == 0) +{ + Console.Error.WriteLine("No input files or directories provided."); + return 64; // EX_USAGE +} + +var options = new PolicyValidationCliOptions +{ + Inputs = inputs, + Strict = strict, + OutputJson = outputJson, +}; + +var cli = new PolicyValidationCli(); +var exitCode = await cli.RunAsync(options, CancellationToken.None); +return exitCode; diff --git a/tools/PolicySchemaExporter/PolicySchemaExporter.csproj b/tools/PolicySchemaExporter/PolicySchemaExporter.csproj new file mode 100644 index 00000000..040cb465 --- /dev/null +++ b/tools/PolicySchemaExporter/PolicySchemaExporter.csproj @@ -0,0 +1,21 @@ + + + + Exe + net10.0 + enable + enable + true + + + + + + + + + + + + + diff --git a/tools/PolicySchemaExporter/Program.cs b/tools/PolicySchemaExporter/Program.cs new file mode 100644 index 00000000..f37a263f --- /dev/null +++ b/tools/PolicySchemaExporter/Program.cs @@ -0,0 +1,48 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.Json.Serialization; +using NJsonSchema; +using NJsonSchema.Generation; +using NJsonSchema.Generation.SystemTextJson; +using Newtonsoft.Json; +using StellaOps.Scheduler.Models; + +var output = args.Length switch +{ + 0 => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "docs", "schemas")), + 1 => Path.GetFullPath(args[0]), + _ => throw new ArgumentException("Usage: dotnet run --project tools/PolicySchemaExporter -- [outputDirectory]") +}; + +Directory.CreateDirectory(output); + +var generatorSettings = new SystemTextJsonSchemaGeneratorSettings +{ + SchemaType = SchemaType.JsonSchema, + DefaultReferenceTypeNullHandling = ReferenceTypeNullHandling.NotNull, + SerializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }, +}; + +var generator = new JsonSchemaGenerator(generatorSettings); + +var exports = ImmutableArray.Create( + (FileName: "policy-run-request.schema.json", Type: typeof(PolicyRunRequest)), + (FileName: "policy-run-status.schema.json", Type: typeof(PolicyRunStatus)), + (FileName: "policy-diff-summary.schema.json", Type: typeof(PolicyDiffSummary)), + (FileName: "policy-explain-trace.schema.json", Type: typeof(PolicyExplainTrace)) +); + +foreach (var export in exports) +{ + var schema = generator.Generate(export.Type); + schema.Title = export.Type.Name; + schema.AllowAdditionalProperties = false; + + var outputPath = Path.Combine(output, export.FileName); + await File.WriteAllTextAsync(outputPath, schema.ToJson(Formatting.Indented) + Environment.NewLine); + Console.WriteLine($"Wrote {outputPath}"); +} diff --git a/tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj b/tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj new file mode 100644 index 00000000..95b4d40b --- /dev/null +++ b/tools/PolicySimulationSmoke/PolicySimulationSmoke.csproj @@ -0,0 +1,14 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/tools/PolicySimulationSmoke/Program.cs b/tools/PolicySimulationSmoke/Program.cs new file mode 100644 index 00000000..8c537e63 --- /dev/null +++ b/tools/PolicySimulationSmoke/Program.cs @@ -0,0 +1,291 @@ +using System.Collections.Immutable; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Policy; + +var scenarioRoot = "samples/policy/simulations"; +string? outputDir = null; + +for (var i = 0; i < args.Length; i++) +{ + var arg = args[i]; + switch (arg) + { + case "--scenario-root": + case "-r": + if (i + 1 >= args.Length) + { + Console.Error.WriteLine("Missing value for --scenario-root."); + return 64; + } + scenarioRoot = args[++i]; + break; + case "--output": + case "-o": + if (i + 1 >= args.Length) + { + Console.Error.WriteLine("Missing value for --output."); + return 64; + } + outputDir = args[++i]; + break; + case "--help": + case "-h": + case "-?": + PrintUsage(); + return 0; + default: + Console.Error.WriteLine($"Unknown argument '{arg}'."); + PrintUsage(); + return 64; + } +} + +if (!Directory.Exists(scenarioRoot)) +{ + Console.Error.WriteLine($"Scenario root '{scenarioRoot}' does not exist."); + return 66; +} + +var scenarioFiles = Directory.GetFiles(scenarioRoot, "scenario.json", SearchOption.AllDirectories); +if (scenarioFiles.Length == 0) +{ + Console.Error.WriteLine($"No scenario.json files found under '{scenarioRoot}'."); + return 0; +} + +var loggerFactory = NullLoggerFactory.Instance; +var snapshotStore = new PolicySnapshotStore( + new NullPolicySnapshotRepository(), + new NullPolicyAuditRepository(), + TimeProvider.System, + loggerFactory.CreateLogger()); +var previewService = new PolicyPreviewService(snapshotStore, loggerFactory.CreateLogger()); + +var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) +{ + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, +}; + +var summary = new List(); +var success = true; + +foreach (var scenarioFile in scenarioFiles.OrderBy(static f => f, StringComparer.OrdinalIgnoreCase)) +{ + var scenarioText = await File.ReadAllTextAsync(scenarioFile); + var scenario = JsonSerializer.Deserialize(scenarioText, serializerOptions); + if (scenario is null) + { + Console.Error.WriteLine($"Failed to deserialize scenario '{scenarioFile}'."); + success = false; + continue; + } + + var repoRoot = Directory.GetCurrentDirectory(); + var policyPath = Path.Combine(repoRoot, scenario.PolicyPath); + if (!File.Exists(policyPath)) + { + Console.Error.WriteLine($"Policy file '{scenario.PolicyPath}' referenced by scenario '{scenario.Name}' does not exist."); + success = false; + continue; + } + + var policyContent = await File.ReadAllTextAsync(policyPath); + var policyFormat = PolicySchema.DetectFormat(policyPath); + var findings = scenario.Findings.Select(ToPolicyFinding).ToImmutableArray(); + var baseline = scenario.Baseline?.Select(ToPolicyVerdict).ToImmutableArray() ?? ImmutableArray.Empty; + + var request = new PolicyPreviewRequest( + ImageDigest: $"sha256:simulation-{scenario.Name}", + Findings: findings, + BaselineVerdicts: baseline, + SnapshotOverride: null, + ProposedPolicy: new PolicySnapshotContent( + Content: policyContent, + Format: policyFormat, + Actor: "ci", + Source: "ci/simulation-smoke", + Description: $"CI simulation for scenario '{scenario.Name}'")); + + var response = await previewService.PreviewAsync(request, CancellationToken.None); + var scenarioResult = EvaluateScenario(scenario, response); + summary.Add(scenarioResult); + + if (!scenarioResult.Success) + { + success = false; + } +} + +if (outputDir is not null) +{ + Directory.CreateDirectory(outputDir); + var summaryPath = Path.Combine(outputDir, "policy-simulation-summary.json"); + await File.WriteAllTextAsync(summaryPath, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true })); +} + +return success ? 0 : 1; + +static void PrintUsage() +{ + Console.WriteLine("Usage: policy-simulation-smoke [--scenario-root ] [--output ]"); + Console.WriteLine("Example: policy-simulation-smoke --scenario-root samples/policy/simulations --output artifacts/policy-simulations"); +} + +static PolicyFinding ToPolicyFinding(ScenarioFinding finding) +{ + var tags = finding.Tags is null ? ImmutableArray.Empty : ImmutableArray.CreateRange(finding.Tags); + var severity = Enum.Parse(finding.Severity, ignoreCase: true); + return new PolicyFinding( + finding.FindingId, + severity, + finding.Environment, + finding.Source, + finding.Vendor, + finding.License, + finding.Image, + finding.Repository, + finding.Package, + finding.Purl, + finding.Cve, + finding.Path, + finding.LayerDigest, + tags); +} + +static PolicyVerdict ToPolicyVerdict(ScenarioBaseline baseline) +{ + var status = Enum.Parse(baseline.Status, ignoreCase: true); + var inputs = baseline.Inputs?.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase) ?? ImmutableDictionary.Empty; + return new PolicyVerdict( + baseline.FindingId, + status, + RuleName: baseline.RuleName, + RuleAction: baseline.RuleAction, + Notes: baseline.Notes, + Score: baseline.Score, + ConfigVersion: baseline.ConfigVersion ?? PolicyScoringConfig.Default.Version, + Inputs: inputs, + QuietedBy: null, + Quiet: false, + UnknownConfidence: null, + ConfidenceBand: null, + UnknownAgeDays: null, + SourceTrust: null, + Reachability: null); +} + +static ScenarioResult EvaluateScenario(PolicySimulationScenario scenario, PolicyPreviewResponse response) +{ + var result = new ScenarioResult(scenario.Name); + if (!response.Success) + { + result.Failures.Add("Preview failed."); + return result with { Success = false, ChangedCount = response.ChangedCount }; + } + + var diffs = response.Diffs.ToDictionary(diff => diff.Projected.FindingId, StringComparer.OrdinalIgnoreCase); + foreach (var expected in scenario.ExpectedDiffs) + { + if (!diffs.TryGetValue(expected.FindingId, out var diff)) + { + result.Failures.Add($"Expected finding '{expected.FindingId}' missing from diff."); + continue; + } + + var projectedStatus = diff.Projected.Status.ToString(); + result.ActualStatuses[expected.FindingId] = projectedStatus; + if (!string.Equals(projectedStatus, expected.Status, StringComparison.OrdinalIgnoreCase)) + { + result.Failures.Add($"Finding '{expected.FindingId}' expected status '{expected.Status}' but was '{projectedStatus}'."); + } + } + + foreach (var diff in diffs.Values) + { + if (!result.ActualStatuses.ContainsKey(diff.Projected.FindingId)) + { + result.ActualStatuses[diff.Projected.FindingId] = diff.Projected.Status.ToString(); + } + } + + var success = result.Failures.Count == 0; + return result with + { + Success = success, + ChangedCount = response.ChangedCount + }; +} + +internal sealed record PolicySimulationScenario +{ + public string Name { get; init; } = "scenario"; + public string PolicyPath { get; init; } = string.Empty; + public List Findings { get; init; } = new(); + public List ExpectedDiffs { get; init; } = new(); + public List? Baseline { get; init; } +} + +internal sealed record ScenarioFinding +{ + public string FindingId { get; init; } = string.Empty; + public string Severity { get; init; } = "Low"; + public string? Environment { get; init; } + public string? Source { get; init; } + public string? Vendor { get; init; } + public string? License { get; init; } + public string? Image { get; init; } + public string? Repository { get; init; } + public string? Package { get; init; } + public string? Purl { get; init; } + public string? Cve { get; init; } + public string? Path { get; init; } + public string? LayerDigest { get; init; } + public string[]? Tags { get; init; } +} + +internal sealed record ScenarioExpectedDiff +{ + public string FindingId { get; init; } = string.Empty; + public string Status { get; init; } = "Pass"; +} + +internal sealed record ScenarioBaseline +{ + public string FindingId { get; init; } = string.Empty; + public string Status { get; init; } = "Pass"; + public string? RuleName { get; init; } + public string? RuleAction { get; init; } + public string? Notes { get; init; } + public double Score { get; init; } + public string? ConfigVersion { get; init; } + public Dictionary? Inputs { get; init; } +} + +internal sealed record ScenarioResult(string ScenarioName) +{ + public bool Success { get; init; } = true; + public int ChangedCount { get; init; } + public List Failures { get; } = new(); + public Dictionary ActualStatuses { get; } = new(StringComparer.OrdinalIgnoreCase); +} + +internal sealed class NullPolicySnapshotRepository : IPolicySnapshotRepository +{ + public Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task GetLatestAsync(CancellationToken cancellationToken = default) => Task.FromResult(null); + + public Task> ListAsync(int limit, CancellationToken cancellationToken = default) + => Task.FromResult>(Array.Empty()); +} + +internal sealed class NullPolicyAuditRepository : IPolicyAuditRepository +{ + public Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task> ListAsync(int limit, CancellationToken cancellationToken = default) + => Task.FromResult>(Array.Empty()); +}