diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md index 5f6519cba..73465303f 100755 --- a/docs/24_OFFLINE_KIT.md +++ b/docs/24_OFFLINE_KIT.md @@ -15,11 +15,15 @@ completely isolated network: | **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU | | **Container images** | `stella-ops`, *Zastava* sidecar, `advisory-ai-web`, and `advisory-ai-worker` (x86‑64 & arm64) | | **Provenance** | Cosign signature, SPDX 2.3 SBOM, in‑toto SLSA attestation | -| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | -| **Delta patches** | Daily diff bundles keep size \< 350 MB | +| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | +| **Delta patches** | Daily diff bundles keep size \< 350 MB | | **Scanner plug-ins** | OS analyzers plus the Node.js, Go, .NET, Python, and Rust language analyzers packaged under `plugins/scanner/analyzers/**` with manifests so Workers load deterministically offline. | -| **Debug store** | `.debug` artefacts laid out under `debug/.build-id//.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. | -| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. | +| **Debug store** | `.debug` artefacts laid out under `debug/.build-id//.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. | +| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. | +| **CLI + Task Packs** | `cli/` binaries from `release/cli`, Task Runner bootstrap (`bootstrap/task-runner/task-runner.yaml.sample`), and task-pack docs under `docs/task-packs/**` + `docs/modules/taskrunner/**`. | +| **Orchestrator/Export/Notifier kits** | Orchestrator service, worker SDK, Postgres snapshot, dashboards (`orchestrator/**`), Export Center bundles (`export-center/**`), Notifier offline packs (`notifier/**`). | +| **Container air-gap bundles** | Any tar/tgz under `containers/` or `images/` (mirrored registries) plus `docs/airgap/mirror-bundles.md`. | +| **Surface.Secrets** | Encrypted secrets bundles and manifests (`surface-secrets/**`) for sealed-mode bootstrap. | **RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an air‑gapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache. @@ -46,8 +50,15 @@ The helper copies `debug/.build-id/**`, validates `debug/debug-manifest.json` ag ## 0.1 · Automated packaging -The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`. -It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set. +The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`. +It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set. + +What it picks up automatically (if present under `--release-dir`): +- `cli/**` → CLI binaries and installers. +- `containers/**` or `images/**` → air-gap container bundles. +- `orchestrator/{service,worker-sdk,postgres,dashboards}/**`. +- `export-center/**`, `notifier/**`, `surface-secrets/**`. +- Docs: `docs/task-packs/**`, `docs/modules/taskrunner/**`, `docs/airgap/mirror-bundles.md`. ```bash python ops/offline-kit/build_offline_kit.py \ diff --git a/docs/api/graph.md b/docs/api/graph.md new file mode 100644 index 000000000..660aa6bf0 --- /dev/null +++ b/docs/api/graph.md @@ -0,0 +1,123 @@ +# Graph API + +Status: Draft (2025-11-26) — aligns with Sprint 0207 Graph API implementation (in-memory seed; RBAC + budgets enforced). + +Base URL: `/api/graph` (examples use relative paths). + +## Common headers +- `X-Stella-Tenant` (required) +- `Authorization: Bearer ` (required) +- `X-Stella-Scopes`: space/comma/semicolon separated scopes + - `/graph/search`: `graph:read` or `graph:query` + - `/graph/query|paths|diff`: `graph:query` + - `/graph/export`: `graph:export` +Content-Type for requests: `application/json` +Streaming responses: `application/x-ndjson` + +## POST /graph/search +Returns NDJSON stream of tiles (`node`, optional `cursor`). + +Body: +```json +{ + "kinds": ["component", "artifact"], + "query": "pkg:npm/", + "filters": { "ecosystem": "npm" }, + "limit": 50, + "cursor": "opaque" +} +``` +Errors: +- 400 `GRAPH_VALIDATION_FAILED` (missing kinds/query/filters) +- 401 missing auth; 403 missing scopes + +## POST /graph/query +Streams nodes, edges (optional), stats, cursor with cost metadata. + +Body: +```json +{ + "kinds": ["component"], + "query": "widget", + "filters": { "tenant": "acme" }, + "includeEdges": true, + "includeStats": true, + "includeOverlays": true, + "limit": 100, + "cursor": "opaque", + "budget": { "tiles": 6000, "nodes": 5000, "edges": 10000 } +} +``` +Error tile if edge budget exceeded: `{ "type":"error","data":{"error":"GRAPH_BUDGET_EXCEEDED",...}}` + +## POST /graph/paths +Finds paths up to depth 6 between sources/targets. + +Body: +```json +{ + "sources": ["gn:acme:component:one"], + "targets": ["gn:acme:component:two"], + "kinds": ["depends_on"], + "maxDepth": 4, + "includeOverlays": false, + "budget": { "tiles": 2000, "nodes": 1500, "edges": 3000 } +} +``` +Response: NDJSON tiles (`node`, `edge`, `stats`, `cursor`). + +## POST /graph/diff +Diff two snapshots. + +Body: +```json +{ + "snapshotA": "snapA", + "snapshotB": "snapB", + "includeEdges": true, + "includeStats": true, + "budget": { "tiles": 2000 } +} +``` +Response tiles: `node`/`edge` with change types, `stats`, optional `cursor`. + +## POST /graph/export +Kicks off an in-memory export job and returns manifest. + +Body: +```json +{ + "format": "ndjson", // ndjson|csv|graphml|png|svg + "includeEdges": true, + "snapshotId": "snapA", // optional + "kinds": ["component"], // optional + "query": "pkg:", // optional + "filters": { "ecosystem": "npm" } +} +``` +Response: +```json +{ + "jobId": "job-123", + "status": "completed", + "format": "ndjson", + "sha256": "...", + "size": 1234, + "downloadUrl": "/graph/export/job-123", + "completedAt": "2025-11-26T00:00:00Z" +} +``` +Download: `GET /graph/export/{jobId}` (returns file, headers include `X-Content-SHA256`). + +## Health +`GET /healthz` → 200 when service is ready; no auth/scopes required. + +## Error envelope +```json +{ "error": "GRAPH_VALIDATION_FAILED", "message": "details", "requestId": "optional" } +``` + +## Notes +- All timestamps UTC ISO-8601. +- Ordering is deterministic; cursors are opaque base64 offsets. +- Overlays use `policy.overlay.v1` and `openvex.v1` payloads. Budgeted tiles reserve room for cursors when `hasMore` is true. diff --git a/docs/api/vuln.md b/docs/api/vuln.md new file mode 100644 index 000000000..703097fbd --- /dev/null +++ b/docs/api/vuln.md @@ -0,0 +1,29 @@ +# Vulnerability API (placeholder) + +Status: Draft (2025-11-26) — awaiting Vuln Explorer v1 surface. This doc reserves the path and headers to align with upcoming releases. + +## Base URL +`/api/vuln` (subject to final routing via API gateway). + +## Common headers +- `X-Stella-Tenant` (required) +- `Authorization: Bearer ` +- `X-Stella-Scopes`: expect `vuln:read` (TBD) and/or `graph:read` when graph-backed queries are invoked. +- `Content-Type: application/json` + +## Planned endpoints (subject to change) +- `POST /vuln/search` — filter vulnerabilities by component (purl/digest), advisory id, status, exploitability (OpenVEX). +- `POST /vuln/impact` — compute impacted assets using Graph overlays; may proxy to Graph API internally. +- `GET /vuln/{id}` — details with references, VEX status, nearest safe version. +- `GET /vuln/{id}/evidence` — raw evidence (SBOM snapshot refs, observations). +- `GET /vuln/kev` — Known Exploited Vulnerabilities view (cached). + +## Error envelope +Follows Graph/Platform standard: +```json +{ "error": "VULN_VALIDATION_FAILED", "message": "details", "requestId": "optional" } +``` + +## Notes +- This placeholder will be updated once Vuln Explorer API is finalized. Keep gateway clients tolerant to minor shape changes until status flips to READY. +- For current graph-backed queries, use `/graph/search` or `/graph/query` (see `docs/api/graph.md`). diff --git a/docs/implplan/SPRINT_0207_0001_0001_graph.md b/docs/implplan/SPRINT_0207_0001_0001_graph.md index 8d1787e3e..007ee8f25 100644 --- a/docs/implplan/SPRINT_0207_0001_0001_graph.md +++ b/docs/implplan/SPRINT_0207_0001_0001_graph.md @@ -31,9 +31,9 @@ | 6 | GRAPH-API-28-006 | DONE (2025-11-26) | GRAPH-API-28-005; POLICY-ENGINE-30-001..003 contracts | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Consume Policy Engine overlay contract and surface advisory/VEX/policy overlays with caching, partial materialization, and explain trace sampling for focused nodes. | | 7 | GRAPH-API-28-007 | DONE (2025-11-26) | GRAPH-API-28-006 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Implement exports (`graphml`, `csv`, `ndjson`, `png`, `svg`) with async job management, checksum manifests, and streaming downloads. | | 8 | GRAPH-API-28-008 | DONE (2025-11-26) | GRAPH-API-28-007 | Graph API + Authority Guilds (`src/Graph/StellaOps.Graph.Api`) | Integrate RBAC scopes (`graph:read`, `graph:query`, `graph:export`), tenant headers, audit logging, and rate limiting. | -| 9 | GRAPH-API-28-009 | TODO | GRAPH-API-28-008 | Graph API + Observability Guilds (`src/Graph/StellaOps.Graph.Api`) | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. | +| 9 | GRAPH-API-28-009 | DONE (2025-11-26) | GRAPH-API-28-008 | Graph API + Observability Guilds (`src/Graph/StellaOps.Graph.Api`) | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. | | 10 | GRAPH-API-28-010 | DONE (2025-11-26) | GRAPH-API-28-009 | Graph API Guild · QA Guild (`src/Graph/StellaOps.Graph.Api`) | Build unit/integration/load tests with synthetic datasets (500k nodes/2M edges), fuzz query validation, verify determinism across runs. | -| 11 | GRAPH-API-28-011 | TODO | GRAPH-API-28-010 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. | +| 11 | GRAPH-API-28-011 | DONE (2025-11-26) | GRAPH-API-28-010 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. | | 12 | GRAPH-INDEX-28-011 | DONE (2025-11-04) | Downstream consumption by API once overlays ready | Graph Indexer Guild (`src/Graph/StellaOps.Graph.Indexer`) | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. | ## Wave Coordination diff --git a/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md b/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md index 0386a463b..196c878df 100644 --- a/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md +++ b/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md @@ -22,8 +22,8 @@ | --- | --- | --- | --- | --- | --- | | P1 | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | DONE (2025-11-22) | Due 2025-11-25 · Accountable: Ops Guild | Ops Guild | Waiting for next demo outputs to review dashboards/runbooks.

Document artefact/deliverable for GRAPH-OPS-0001 and publish location so downstream tasks can proceed. Prep artefact: `docs/modules/graph/prep/2025-11-20-ops-0001-prep.md`. | | 1 | GRAPH-ENG-0001 | DONE | Synced docs to Sprint 0141 rename on 2025-11-17 | Module Team | Keep module milestones in sync with `/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md` and related files; update references and note deltas. | -| 2 | GRAPH-DOCS-0002 | BLOCKED | Await DOCS-GRAPH-24-003 cross-links | Docs Guild | Add API/query doc cross-links once DOCS-GRAPH-24-003 lands. | -| 3 | GRAPH-OPS-0001 | TODO | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | Ops Guild | Review graph observability dashboards/runbooks after the next sprint demo; capture updates in runbooks. | +| 2 | GRAPH-DOCS-0002 | DONE (2025-11-26) | DOCS-GRAPH-24-003 delivered | Docs Guild | Add API/query doc cross-links once DOCS-GRAPH-24-003 lands. | +| 3 | GRAPH-OPS-0001 | DONE (2025-11-26) | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | Ops Guild | Review graph observability dashboards/runbooks after the next sprint demo; capture updates in runbooks. | ## Execution Log | Date (UTC) | Update | Owner | @@ -35,6 +35,8 @@ | 2025-11-17 | Normalised sprint to standard template; renamed from SPRINT_321_docs_modules_graph.md. | Docs | | 2025-11-22 | Marked all PREP tasks to DONE per directive; evidence to be verified. | Project Mgmt | | 2025-11-22 | PREP-GRAPH-OPS-0001 done; moved GRAPH-OPS-0001 to TODO pending next demo outputs. | Project Mgmt | +| 2025-11-26 | GRAPH-DOCS-0002 completed: added `architecture-index.md` plus README cross-link covering data model, ingestion pipeline, overlays, events, API/metrics pointers. | Docs Guild | +| 2025-11-26 | GRAPH-OPS-0001 completed: added ops/runbook guidance to `docs/modules/graph/README.md` (health checks, key metrics, alerts, triage steps) and linked Grafana dashboard import path. | Ops Guild | ## Decisions & Risks - Cross-links blocked on DOCS-GRAPH-24-003; track before marking GRAPH-DOCS-0002 done. diff --git a/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md b/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md index 5dfae5389..73905c8e5 100644 --- a/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md +++ b/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md @@ -20,7 +20,7 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | ZASTAVA-REACH-201-001 | DOING (2025-11-26) | Need runtime symbol sampling design; align with GAP-ZAS-002 | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. | +| 1 | ZASTAVA-REACH-201-001 | DONE (2025-11-26) | Runtime facts emitter shipped in Observer | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. | | 9 | GAP-ZAS-002 | BLOCKED (2025-11-26) | Align with task 1; runtime NDJSON schema | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint 0401 lands ingestion. | | 2 | SCAN-REACH-201-002 | DOING (2025-11-23) | Schema published: `docs/reachability/runtime-static-union-schema.md` (v0.1). Implement emitters against CAS layout. | Scanner Worker Guild | Ship language-aware static lifters (JVM, .NET/Roslyn+IL, Go SSA, Node/Deno TS AST, Rust MIR, Swift SIL, shell/binary analyzers) in Scanner Worker; emit canonical SymbolIDs, CAS-stored graphs, and attach reachability tags to SBOM components. | | 3 | SIGNALS-REACH-201-003 | DONE (2025-11-25) | Consume schema `docs/reachability/runtime-static-union-schema.md`; wire ingestion + CAS storage. | Signals Guild | Extend Signals ingestion to accept the new multi-language graphs + runtime facts, normalize into `reachability_graphs` CAS layout, and expose retrieval APIs for Policy/CLI. | @@ -29,13 +29,14 @@ | 6 | DOCS-REACH-201-006 | DONE (2025-11-26) | Requires outputs from 1–5 | Docs Guild | Author the reachability doc set (`docs/reachability/reachability.md`, `callgraph-formats.md`, `runtime-facts.md`, CLI/UI appendices) plus update Zastava + Replay guides with the new evidence and operator workflows. | | 7 | QA-REACH-201-007 | DONE (2025-11-25) | Move fixtures + create evaluator harness | QA Guild | Integrate `reachbench-2025-expanded` fixture pack under `tests/reachability/fixtures/`, add evaluator harness tests that validate reachable vs unreachable cases, and wire CI guidance for deterministic runs. | | 8 | GAP-SCAN-001 | BLOCKED (2025-11-26) | Richgraph-v1 schema not final; Scanner workspace currently dirty, unsafe to land symbolizer changes. | Scanner Worker Guild | Implement binary/language symbolizers that emit `richgraph-v1` payloads with canonical SymbolIDs and `code_id` anchors, persist graphs to CAS via `StellaOps.Scanner.Reachability`, and refresh analyzer docs/fixtures. | -| 9 | GAP-ZAS-002 | BLOCKED (2025-11-26) | Dirty Zastava tree; need clean state to add runtime NDJSON emitter without clobbering user changes. | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint 0401 lands ingestion. | +| 9 | GAP-ZAS-002 | DONE (2025-11-26) | Runtime NDJSON emitter merged; config enables callgraph-linked facts | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint 0401 lands ingestion. | | 10 | SIGNALS-UNKNOWN-201-008 | DONE (2025-11-26) | Needs schema alignment with reachability store | Signals Guild | Implement Unknowns Registry ingestion and storage for unresolved symbols/edges or purl gaps; expose `/unknowns/*` APIs, feed `unknowns_pressure` into scoring, and surface metrics/hooks for Policy/UI. | | 11 | GRAPH-PURL-201-009 | BLOCKED (2025-11-26) | Depends on GAP-SCAN-001 and final richgraph-v1; pending stable symbolizer outputs. | Scanner Worker Guild · Signals Guild | Define and implement purl + symbol-digest edge annotations in `richgraph-v1`, update CAS metadata and SBOM join logic, and round-trip through Signals/Policy/CLI explainers. | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-26 | Implemented runtime facts emitter in `StellaOps.Zastava.Observer` (callgraph-aware NDJSON publish + subject derivation); added reachability options and unit tests; set 201-001 and GAP-ZAS-002 to DONE. | Zastava Observer Guild | | 2025-11-26 | Drafted runtime sampler runbook updates (config knobs, sampler rules, CAS trace pointers) in `docs/runbooks/reachability-runtime.md`; set ZASTAVA-REACH-201-001 to DOING while code waits on clean Zastava workspace. | Zastava Observer Guild | | 2025-11-18 | Normalised sprint to standard template; renamed from SPRINT_400_runtime_facts_static_callgraph_union.md. | Docs | | 2025-11-23 | Published runtime/static union schema v0.1 at `docs/reachability/runtime-static-union-schema.md`; moved 201-002..201-005 to TODO. | Project Mgmt | @@ -61,7 +62,7 @@ - Offline posture: ensure reachability pipelines avoid external downloads; rely on sealed/mock bundles. - Unknowns Registry shipped (201-008): unknowns pressure applied to scoring; monitor schema adjustments from policy team for purl/digest merge (201-009) to avoid churn. - purl + symbol-digest edge schema (201-009) depends on `richgraph-v1` finalization; may require updates to SBOM resolver and CLI explain flows. -- Runtime sampler code pending clean Zastava workspace; runbook updated so implementation can follow once tree is clean. +- Runtime sampler shipped in Observer; ensure `Reachability:CallgraphId` and Signals endpoint are configured per runbook before enabling in production. ## Next Checkpoints - 2025-11-19 · Runtime/static schema alignment session (Symbols, CAS layout). Owner: Signals Guild. diff --git a/docs/implplan/SPRINT_0513_0001_0001_provenance.md b/docs/implplan/SPRINT_0513_0001_0001_provenance.md index 7a93a7891..695025a2e 100644 --- a/docs/implplan/SPRINT_0513_0001_0001_provenance.md +++ b/docs/implplan/SPRINT_0513_0001_0001_provenance.md @@ -62,6 +62,7 @@ ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-26 | Attempted `dotnet test ...Attestation.Tests.csproj -c Release --filter FullyQualifiedName!~RotatingSignerTests`; build fanned out and was cancelled locally after long MSBuild churn. CI runner still needed; tasks PROV-OBS-54-001/54-002 remain BLOCKED. | Implementer | | 2025-11-25 | Retried build locally: `dotnet build src/Provenance/StellaOps.Provenance.Attestation/StellaOps.Provenance.Attestation.csproj -c Release` succeeded in 1.6s. Subsequent `dotnet build --no-restore` on Attestation.Tests still fans out across Concelier dependencies (static graph) and was cancelled; test run remains blocked. Need CI/filtered graph to validate PROV-OBS-53-002/54-001. | Implementer | | 2025-11-25 | Attempted `dotnet test src/Provenance/__Tests/StellaOps.Provenance.Attestation.Tests/StellaOps.Provenance.Attestation.Tests.csproj -c Release`; build fanned out across Concelier dependencies and was cancelled after 63.5s. PROV-OBS-54-001 kept BLOCKED pending CI rerun on faster runner. | Implementer | | 2025-11-22 | PROV-OBS-54-002 delivered: global tool `stella-forensic-verify` updated with signed-at/not-after/skew options, deterministic JSON output, README packaging steps, and tests. | Implementer | diff --git a/docs/implplan/SPRINT_304_docs_tasks_md_iv.md b/docs/implplan/SPRINT_304_docs_tasks_md_iv.md index 84b9e1cf8..7cf6e8a20 100644 --- a/docs/implplan/SPRINT_304_docs_tasks_md_iv.md +++ b/docs/implplan/SPRINT_304_docs_tasks_md_iv.md @@ -17,12 +17,19 @@ DOCS-FORENSICS-53-002 | TODO | Release `/docs/forensics/provenance-attestation.m DOCS-FORENSICS-53-003 | TODO | Publish `/docs/forensics/timeline.md` with schema, event kinds, filters, query examples, and imposed rule banner. Dependencies: DOCS-FORENSICS-53-002. | Docs Guild, Timeline Indexer Guild (docs) DOCS-GRAPH-24-001 | TODO | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Docs Guild, UI Guild (docs) DOCS-GRAPH-24-002 | TODO | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Docs Guild, UI Guild (docs) -DOCS-GRAPH-24-003 | TODO | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Docs Guild, SBOM Service Guild (docs) +DOCS-GRAPH-24-003 | DONE (2025-11-26) | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Docs Guild, SBOM Service Guild (docs) DOCS-GRAPH-24-004 | TODO | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Docs Guild, BE-Base Platform Guild (docs) -DOCS-GRAPH-24-005 | TODO | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Docs Guild, DevEx/CLI Guild (docs) +DOCS-GRAPH-24-005 | DONE (2025-11-26) | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Docs Guild, DevEx/CLI Guild (docs) DOCS-GRAPH-24-006 | TODO | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Docs Guild, Policy Guild (docs) DOCS-GRAPH-24-007 | TODO | Produce `/docs/migration/graph-parity.md` with rollout plan, parity checks, fallback guidance. Dependencies: DOCS-GRAPH-24-006. | Docs Guild, DevOps Guild (docs) DOCS-PROMO-70-001 | TODO | Publish `/docs/release/promotion-attestations.md` describing the promotion workflow (CLI commands, Signer/Attestor integration, offline verification) and update `/docs/forensics/provenance-attestation.md` with the new predicate. Dependencies: PROV-OBS-53-003, CLI-PROMO-70-002. | Docs Guild, Provenance Guild (docs) DOCS-DETER-70-002 | TODO | Document the scanner determinism score process (`determinism.json` schema, CI harness, replay instructions) under `/docs/modules/scanner/determinism-score.md` and add a release-notes template entry. Dependencies: SCAN-DETER-186-010, DEVOPS-SCAN-90-004. | Docs Guild, Scanner Guild (docs) DOCS-SYMS-70-003 | TODO | Author symbol-server architecture/spec docs (`docs/specs/symbols/SYMBOL_MANIFEST_v1.md`, API reference, bundle guide) and update reachability guides with symbol lookup workflow and tenant controls. Dependencies: SYMS-SERVER-401-011, SYMS-INGEST-401-013. | Docs Guild, Symbols Guild (docs) DOCS-ENTROPY-70-004 | TODO | Publish entropy analysis documentation (scoring heuristics, JSON schemas, policy hooks, UI guidance) under `docs/modules/scanner/entropy.md` and update trust-lattice references. Dependencies: SCAN-ENTROPY-186-011/012, POLICY-RISK-90-001. | Docs Guild, Scanner Guild (docs) + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-26 | DOCS-GRAPH-24-003 completed: created `docs/modules/graph/architecture-index.md` covering data model, ingestion pipeline, overlays/caches, events, and API/metrics pointers; unblocks downstream graph doc tasks. | Docs Guild | +| 2025-11-26 | DOCS-GRAPH-24-004 completed: published `docs/api/graph.md` (search/query/paths/diff/export, headers, budgets, errors) and placeholder `docs/api/vuln.md`; next tasks can link to these APIs. | Docs Guild | +| 2025-11-26 | DOCS-GRAPH-24-005 completed: refreshed CLI guide (`docs/modules/cli/guides/graph-and-vuln.md`) with commands, budgets, paging, export, exit codes; unblocks 24-006. | Docs Guild | diff --git a/docs/implplan/SPRINT_508_ops_offline_kit.md b/docs/implplan/SPRINT_508_ops_offline_kit.md index 8d6ac3956..70693b9fa 100644 --- a/docs/implplan/SPRINT_508_ops_offline_kit.md +++ b/docs/implplan/SPRINT_508_ops_offline_kit.md @@ -7,10 +7,19 @@ Depends on: Sprint 100.A - Attestor, Sprint 110.A - AdvisoryAI, Sprint 120.A - A Summary: Ops & Offline focus on Ops Offline Kit). Task ID | State | Task description | Owners (Source) --- | --- | --- | --- -CLI-PACKS-43-002 | TODO | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit) +CLI-PACKS-43-002 | DONE (2025-11-26) | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit) DEVOPS-OFFLINE-17-004 | DONE (2025-11-23) | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild, DevOps Guild (ops/offline-kit) -DEVOPS-OFFLINE-34-006 | TODO | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit) -DEVOPS-OFFLINE-37-001 | TODO | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit) -DEVOPS-OFFLINE-37-002 | TODO | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit) -OFFLINE-CONTAINERS-46-001 | TODO | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline Kit Guild, Deployment Guild (ops/offline-kit) -OPS-SECRETS-02 | TODO | Add Surface.Secrets bundles (encrypted creds, manifests) to Offline Kit packaging plus verification script. Dependencies: OPS-SECRETS-02. | Offline Kit Guild, DevOps Guild (ops/offline-kit) +DEVOPS-OFFLINE-34-006 | DONE (2025-11-26) | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit) +DEVOPS-OFFLINE-37-001 | DONE (2025-11-26) | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit) +DEVOPS-OFFLINE-37-002 | DONE (2025-11-26) | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit) +OFFLINE-CONTAINERS-46-001 | DONE (2025-11-26) | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline Kit Guild, Deployment Guild (ops/offline-kit) +OPS-SECRETS-02 | DONE (2025-11-26) | Add Surface.Secrets bundles (encrypted creds, manifests) to Offline Kit packaging plus verification script. Dependencies: OPS-SECRETS-02. | Offline Kit Guild, DevOps Guild (ops/offline-kit) + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-26 | Wired Offline Kit packaging to include CLI binaries (release/cli), Task Runner bootstrap config, and task-pack docs; updated `test_build_offline_kit.py` to cover new artefacts. Marked CLI-PACKS-43-002 DONE. | Implementer | +| 2025-11-26 | Added container bundle pickup (release/containers/images) and mirrored registry doc copy; updated offline kit test coverage; marked OFFLINE-CONTAINERS-46-001 DONE. | Implementer | +| 2025-11-26 | Added orchestrator (service, worker SDK, postgres, dashboards), Export Center bundles, Notifier offline packs, and Surface.Secrets bundles to packaging; expanded offline kit unit test accordingly. Marked DEVOPS-OFFLINE-34-006/37-001/37-002 and OPS-SECRETS-02 DONE. | Implementer | +| 2025-11-26 | Updated Offline Kit doc (`docs/24_OFFLINE_KIT.md`) to describe newly bundled assets (CLI/task packs, orchestrator/export/notifier kits, container bundles, Surface.Secrets) and documented release-dir auto-pickup rules. | Implementer | +| 2025-11-23 | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild | diff --git a/docs/implplan/tasks-all.md b/docs/implplan/tasks-all.md index 17018a9ff..1894f64b2 100644 --- a/docs/implplan/tasks-all.md +++ b/docs/implplan/tasks-all.md @@ -696,8 +696,8 @@ | DOCS-GRAPH-24-001 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Graph Guild | docs/modules/graph | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Wait for GRAP0101 contract freeze | DOGR0101 | | DOCS-GRAPH-24-002 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · UI Guild | docs/modules/graph | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Needs SBOM/VEX dataflow confirmation (PLLG0104) | DOGR0101 | | DOCS-GRAPH-24-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Unblocked: SBOM join spec delivered with CARTO-GRAPH-21-002 (2025-11-17). | DOGR0101 | -| DOCS-GRAPH-24-004 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/modules/graph | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 | -| DOCS-GRAPH-24-005 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Wait for CLI samples from CLCI0109 | DOGR0101 | +| DOCS-GRAPH-24-004 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/api/graph.md; docs/api/vuln.md | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 | +| DOCS-GRAPH-24-005 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | — | DOGR0101 | | DOCS-GRAPH-24-006 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Policy Guild | docs/modules/graph | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Needs policy outputs from PLVL0102 | DOGR0101 | | DOCS-GRAPH-24-007 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevOps Guild | docs/modules/graph | Produce `/docs/migration/graph-parity.md` with rollout plan, parity checks, fallback guidance. Dependencies: DOCS-GRAPH-24-006. | Depends on DVDO0108 deployment notes | DOGR0101 | | DOCS-INSTALL-44-001 | BLOCKED | 2025-11-25 | SPRINT_305_docs_tasks_md_v | Docs Guild · Deployment Guild | docs/install | Publish `/docs/install/overview.md` and `/docs/install/compose-quickstart.md` with imposed rule line and copy-ready commands. | Blocked: waiting on DVPL0101 compose schema + service list/version pins | DOIS0101 | @@ -1099,14 +1099,14 @@ | GRAPH-API-28-011 | DONE (2025-11-26) | 2025-11-26 | SPRINT_0207_0001_0001_graph | Graph API Guild | src/Graph/StellaOps.Graph.Api | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. Dependencies: GRAPH-API-28-010. | GRAPH-API-28-009 | GRAPI0101 | | GRAPH-CAS-401-001 | TODO | | SPRINT_0401_0001_0001_reachability_evidence_chain | Scanner Worker Guild | `src/Scanner/StellaOps.Scanner.Worker` | Finalize richgraph schema (`richgraph-v1`), emit canonical SymbolIDs, compute graph hash (BLAKE3), and store CAS manifests under `cas://reachability/graphs/{sha256}`. Update Scanner Worker adapters + fixtures. | Depends on #1 | CASC0101 | | GRAPH-DOCS-0001 | DONE (2025-11-05) | 2025-11-05 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Validate that graph module README/diagrams reflect the latest overlay + snapshot updates. | GRAPI0101 evidence | GRDG0101 | -| GRAPH-DOCS-0002 | TODO | 2025-11-05 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Pending DOCS-GRAPH-24-003 to add API/query doc cross-links | GRAPI0101 outputs | GRDG0101 | +| GRAPH-DOCS-0002 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Pending DOCS-GRAPH-24-003 to add API/query doc cross-links | GRAPI0101 outputs | GRDG0101 | | GRAPH-ENG-0001 | TODO | | SPRINT_321_docs_modules_graph | Module Team | docs/modules/graph | Keep module milestones in sync with `/docs/implplan/SPRINT_141_graph.md` and related files. | GRSC0101 | GRDG0101 | | GRAPH-INDEX-28-007 | DOING | | SPRINT_0140_0001_0001_runtime_signals | — | | Running on scanner surface mock bundle v1; will validate again once real caches drop. | — | ORGR0101 | | GRAPH-INDEX-28-008 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Incremental update/backfill pipeline depends on 28-007 artifacts; retry/backoff plumbing sketched but blocked. | — | ORGR0101 | | GRAPH-INDEX-28-009 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Test/fixture/chaos coverage waits on earlier jobs to exist so determinism checks have data. | — | ORGR0101 | | GRAPH-INDEX-28-010 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Packaging/offline bundles paused until upstream graph jobs are available to embed. | — | ORGR0101 | | GRAPH-INDEX-28-011 | TODO | 2025-11-04 | SPRINT_0207_0001_0001_graph | Graph Index Guild | src/Graph/StellaOps.Graph.Indexer | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. Dependencies: GRAPH-INDEX-28-002..006. | GRSC0101 outputs | GRIX0101 | -| GRAPH-OPS-0001 | TODO | | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | +| GRAPH-OPS-0001 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | | HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild (ops/deployment) | ops/deployment | | | GRIX0101 | | HELM-45-002 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Security Guild (ops/deployment) | ops/deployment | Add TLS/Ingress, NetworkPolicy, PodSecurityContexts, Secrets integration (external secrets), and document security posture. Dependencies: HELM-45-001. | | GRIX0101 | | HELM-45-003 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Observability Guild (ops/deployment) | ops/deployment | Implement HPA, PDB, readiness gates, Prometheus scraping annotations, OTel configuration hooks, and upgrade hooks. Dependencies: HELM-45-002. | | GRIX0101 | @@ -2905,7 +2905,7 @@ | DOCS-FORENSICS-53-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Timeline Indexer Guild | docs/modules/evidence-locker/forensics.md | Publish `/docs/forensics/timeline.md` with schema, event kinds, filters, query examples, and imposed rule banner. Dependencies: DOCS-FORENSICS-53-002. | Requires timeline indexer export from 055_AGIM0101 | DOEL0101 | | DOCS-GRAPH-24-001 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Graph Guild | docs/modules/graph | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Wait for GRAP0101 contract freeze | DOGR0101 | | DOCS-GRAPH-24-002 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · UI Guild | docs/modules/graph | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Needs SBOM/VEX dataflow confirmation (PLLG0104) | DOGR0101 | -| DOCS-GRAPH-24-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Blocked on SBOM join spec from CARTO-GRAPH-21-002 | DOGR0101 | +| DOCS-GRAPH-24-003 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Unblocked: SBOM join spec delivered with CARTO-GRAPH-21-002 (2025-11-17). | DOGR0101 | | DOCS-GRAPH-24-004 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/modules/graph | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 | | DOCS-GRAPH-24-005 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Wait for CLI samples from CLCI0109 | DOGR0101 | | DOCS-GRAPH-24-006 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Policy Guild | docs/modules/graph | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Needs policy outputs from PLVL0102 | DOGR0101 | @@ -3319,7 +3319,7 @@ | GRAPH-INDEX-28-009 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Test/fixture/chaos coverage waits on earlier jobs to exist so determinism checks have data. | — | ORGR0101 | | GRAPH-INDEX-28-010 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Packaging/offline bundles paused until upstream graph jobs are available to embed. | — | ORGR0101 | | GRAPH-INDEX-28-011 | TODO | 2025-11-04 | SPRINT_0207_0001_0001_graph | Graph Index Guild | src/Graph/StellaOps.Graph.Indexer | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. Dependencies: GRAPH-INDEX-28-002..006. | GRSC0101 outputs | GRIX0101 | -| GRAPH-OPS-0001 | TODO | | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | +| GRAPH-OPS-0001 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | | HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild (ops/deployment) | ops/deployment | | | GRIX0101 | | HELM-45-002 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Security Guild (ops/deployment) | ops/deployment | Add TLS/Ingress, NetworkPolicy, PodSecurityContexts, Secrets integration (external secrets), and document security posture. Dependencies: HELM-45-001. | | GRIX0101 | | HELM-45-003 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Observability Guild (ops/deployment) | ops/deployment | Implement HPA, PDB, readiness gates, Prometheus scraping annotations, OTel configuration hooks, and upgrade hooks. Dependencies: HELM-45-002. | | GRIX0101 | diff --git a/docs/migration/exception-governance.md b/docs/migration/exception-governance.md new file mode 100644 index 000000000..74e376888 --- /dev/null +++ b/docs/migration/exception-governance.md @@ -0,0 +1,64 @@ +# Exception Governance Migration Guide + +Status: Draft (2025-11-26) — aligns with Excititor VEX/exception flows and Doc task DOCS-EXC-25-007. + +## Why this migration +- Retire legacy suppressions/waivers and move to policy/VEX-first exceptions with provenance. +- Provide auditable, reversible exceptions with rollout/rollback plans and notifications. + +## Target state +- Exceptions are recorded as: + - VEX statements (OpenVEX) referencing findings and components. + - Policy overrides (simulator) with scope, expiry, and rationale. + - Optional authority approval attestation (DSSE) per exception bundle. +- Persistence: Excititor stores exception records with tenant, source, scope, expiry, approver, evidence links. +- Propagation: + - Graph overlays surface exception status on nodes/edges. + - Notify emits `exception.created/expired` events. + - CLI/Console consume exception status via `/exceptions` API. + +## Migration steps +1) **Freeze legacy inputs** + - Disable new legacy suppressions in UI/CLI. + - Mark existing suppressions read-only; export CSV/NDJSON snapshot. +2) **Export legacy suppressions** + - Run `exc suppressions export --format ndjson --with-rationale --with-expiry`. + - Store exports with SHA256 and DSSE envelope in `exceptions/export-YYYYMMDD/`. +3) **Transform to VEX/policy overrides** + - Convert each suppression to OpenVEX statement: + - `status`: `not_affected` or `under_investigation`. + - `justification`: map legacy reason → VEX justification code. + - `impact`: optional; include nearest safe version if known. + - Generate policy override record with: + - `scope` (component, environment, service). + - `expiresAt` (carry forward or set 30/90d). + - `rationale` from legacy note. +4) **Import** + - Use Excititor `/exceptions/import` (or CLI `exc exceptions import`) to load transformed NDJSON. + - Verify import report: counts by status, rejected items, conflicts. +5) **Notify & rollout** + - Notify downstream systems: Graph overlays refresh, Scanner/Vuln Explorer respect VEX, Policy Engine caches reload. + - Announce change freeze window and rollback plan. + +## Rollback plan +- Keep legacy suppressions export. +- If import fails or downstream errors: + - Disable new exception endpoints feature flag. + - Re-enable legacy suppressions (read-only → writable) temporarily. + - Investigate rejected items; re-run transform/import after fix. + +## Notifications +- When migrating: send summary to tenants with counts per status and upcoming expiries. +- Ongoing: Notify events `exception.created`, `exception.expired`, `exception.rejected` with tenant + scope. + +## Validation checklist +- [ ] Legacy suppressions export stored with SHA256 + DSSE. +- [ ] Transform script maps all legacy reasons → VEX justification codes. +- [ ] Import dry-run produces zero rejects or documented rejects with reasons. +- [ ] Overlay/Console shows exceptions on affected components. +- [ ] Rollback tested on staging. + +## References +- Excititor module: `docs/modules/excititor/architecture.md`, `vex_observations.md`. +- Policy simulator: `docs/modules/policy/architecture.md` (POLICY-ENGINE-30-001..003). +- Graph overlays: `docs/modules/graph/architecture-index.md`. diff --git a/docs/modules/cli/guides/graph-and-vuln.md b/docs/modules/cli/guides/graph-and-vuln.md new file mode 100644 index 000000000..aad378531 --- /dev/null +++ b/docs/modules/cli/guides/graph-and-vuln.md @@ -0,0 +1,82 @@ +# CLI Guide · Graph & Vulnerability + +Status: Draft (2025-11-26) — reflects current Graph API surface; Vuln API pending (see docs/api/vuln.md). + +## Prereqs +- `stellaops-cli` built from current repo. +- Auth token with scopes: `graph:read`, `graph:query`, `graph:export` (and `vuln:read` when Vuln API lands). +- Tenant header is required for all calls (`--tenant`). + +## Commands + +### Search graph +```bash +stella graph search \ + --tenant acme \ + --kinds component \ + --query "pkg:npm/" \ + --limit 20 \ + --format ndjson +``` +Outputs NDJSON tiles (`node`, optional `cursor`). + +### Query graph with budgets +```bash +stella graph query \ + --tenant acme \ + --kinds component \ + --query widget \ + --include-edges \ + --budget-tiles 200 \ + --budget-nodes 150 \ + --budget-edges 100 \ + --format ndjson +``` +Returns nodes/edges/stats/cursor with cost metadata. If edge budget is exceeded, an `error` tile with `GRAPH_BUDGET_EXCEEDED` is returned. + +### Paths +```bash +stella graph paths \ + --tenant acme \ + --source gn:acme:component:one \ + --target gn:acme:component:two \ + --max-depth 4 \ + --include-edges \ + --format ndjson +``` + +### Diff snapshots +```bash +stella graph diff \ + --tenant acme \ + --snapshot-a snapA \ + --snapshot-b snapB \ + --include-edges \ + --format ndjson +``` + +### Export graph +```bash +stella graph export \ + --tenant acme \ + --format ndjson \ + --include-edges \ + --kinds component \ + --query "pkg:npm/" \ + --out graph-export.ndjson +``` +Manifest is returned with jobId, sha256, size, and download URL; CLI downloads to `--out`. + +### Vuln lookup (placeholder) +Pending Vuln Explorer API; for now use graph search/query to inspect impacted components. See `docs/api/vuln.md`. + +## Exit codes +- `0` success +- `2` validation error (missing kinds/tenant/scopes) +- `3` budget exceeded (error tile received) +- `4` network/auth failure + +## Tips +- Add `--format table` (when available) for quick summaries; default is NDJSON. +- Use `--cursor` from a previous call to page results. +- To sample overlays, pass `--include-overlays` (budget permitting). diff --git a/docs/modules/graph/README.md b/docs/modules/graph/README.md index f5b1a4778..d84fd3711 100644 --- a/docs/modules/graph/README.md +++ b/docs/modules/graph/README.md @@ -35,11 +35,26 @@ Graph Indexer + Graph API build the tenant-scoped knowledge graph that powers bl - Logs/traces: structured ETL logs, query planner traces, WebGL interaction telemetry (once UI lands). - Offline bundles: deterministic `nodes.jsonl`, `edges.jsonl`, overlay manifests + DSSE signatures, consumable by Export Center and CLI mirroring. +## Operations & runbook (Sprint 030) +- Dashboards: import `Observability/graph-api-grafana.json` (panels for latency, budget denials, overlay cache ratio, export latency). Apply tenant filter in every panel. +- Health checks: `/healthz` should be 200; search/query/paths/diff/export endpoints require `X-Stella-Tenant`, `Authorization`, and scopes (`graph:read/query/export`). +- Key metrics (new): + - `graph_tile_latency_seconds` histogram (label `route`); alert when p95 > 1.5s for 5m. + - `graph_query_budget_denied_total` counter (label `reason`); investigate spikes (>50 in 5m). + - `graph_overlay_cache_hits_total` / `graph_overlay_cache_misses_total`; watch miss ratio > 0.4 for 10m. + - `graph_export_latency_seconds` histogram (label `format`); alert when p95 > 2s for ndjson/graphml. +- Triage playbook: + - Budget denials: lower default edges/nodes budget or guide callers to request smaller scopes; verify overlay includes are truly required. + - Overlay cache misses: ensure cache TTL is ≥5m; check overlay service connectivity to Policy Engine; warm cache by replaying recent hot nodes. + - Export slowness: reduce export `Limit`, offload PNG/SVG to worker, and confirm disk I/O headroom. + - If alerts fire, capture tenant, route, cursor/budget values, and recent deploy SHA in incident note. + ## Key docs & updates - [`architecture.md`](architecture.md) — inputs, pipelines, APIs, storage choices, observability, offline handling. - [`implementation_plan.md`](implementation_plan.md) — phased delivery roadmap, work breakdown, risks, test strategy. - [`schema.md`](schema.md) — canonical node/edge schema and attribute dictionary (keep in sync with indexer code). - Updates: `docs/updates/2025-10-26-scheduler-graph-jobs.md`, `docs/updates/2025-10-26-authority-graph-scopes.md`, `docs/updates/2025-10-30-devops-governance.md` for the latest decisions/dependencies. +- Index: see `architecture-index.md` for data model, ingestion pipeline, overlays/caches, events, and API/observability pointers. ## Epic alignment - **Epic 5 – SBOM Graph Explorer:** Graph Indexer, Graph API, saved queries, overlays, Console/CLI experiences, Offline Kit parity. diff --git a/docs/modules/graph/architecture-index.md b/docs/modules/graph/architecture-index.md new file mode 100644 index 000000000..8d6d88863 --- /dev/null +++ b/docs/modules/graph/architecture-index.md @@ -0,0 +1,48 @@ +# Graph Module · Architecture Index + +Status: Draft (2025-11-26) — aligns with Sprint 0141 (Graph Indexer) and Sprint 0207 (Graph API). + +## Data model (canonical) +- Nodes/edges follow `docs/modules/graph/schema.md`. +- Tenancy: every node/edge/snapshot/event carries `tenant`. +- Snapshots: immutable bundles of nodes + edges; overlays stored separately (`graph_overlays_cache`) to avoid mutating historical snapshots. + +## Ingestion pipeline +1) **SBOM snapshots** from SBOM Service → Graph Indexer (`graph_snapshots`, `graph_nodes`, `graph_edges`). +2) **Advisory/VEX** from Concelier/Excititor → edges and overlay seeds. +3) **Policy overlays** from Policy Engine (POLICY-ENGINE-30-001..003) → cached per node in `graph_overlays_cache`. +4) **Runtime/signals** (future) → additional edges/attributes; stored alongside snapshot edges with tenant guard. + +## Overlays & caches +- Overlay types: policy (`policy.overlay.v1`), vex (`openvex.v1`), analytics (clusters/centrality). +- Cache TTL: 5–10 minutes recommended; deterministic IDs (SHA256 over tenant|node|kind). +- Analytics overlays emitted as NDJSON (`overlays/clusters.ndjson`, `overlays/centrality.ndjson`) with stable ordering. + +## Events & change streams +- Inputs: `sbom.snapshot.created`, `advisory.observation.updated@1`, `sbom.version.created`, `sbom.asset.updated`. +- Outputs: Graph Indexer change-stream/backfill emits node/edge upserts and overlay refresh notifications for Graph API caches. +- Downstream consumers: Scheduler (recompute jobs), Graph API (query caches), Export Center (offline bundles). + +## APIs (reader surfaces) +- Graph API (Sprint 0207): `/graph/search`, `/graph/query`, `/graph/paths`, `/graph/diff`, `/graph/export`. +- Saved queries and snapshots exposed through Graph API; RBAC scopes enforced (`graph:read/query/export`). +- Health check: `/healthz` (200 when service ready). +See also `docs/api/graph.md` (pending) for full endpoint parameters and error envelopes. + +## Observability (current metrics) +- `graph_tile_latency_seconds{route}` +- `graph_query_budget_denied_total{reason}` +- `graph_overlay_cache_hits_total`, `graph_overlay_cache_misses_total` +- `graph_export_latency_seconds{format}` +- Ingest/backfill metrics live in Indexer (Sprint 0141). + +## Offline / bundles +- Exports: deterministic NDJSON (`nodes.jsonl`, `edges.jsonl`, overlays) with SHA256 manifest and optional DSSE. +- Air-gap friendly: no external calls; rely on cached schemas and local snapshots. + +## References +- `architecture.md` — deep dive. +- `implementation_plan.md` — roadmap and risks. +- `schema.md` — canonical node/edge shapes. +- `docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md` — Indexer sprint. +- `docs/implplan/SPRINT_0207_0001_0001_graph.md` — Graph API sprint. diff --git a/ops/devops/release/__pycache__/build_release.cpython-312.pyc b/ops/devops/release/__pycache__/build_release.cpython-312.pyc index 798db2d64..1dab25409 100644 Binary files a/ops/devops/release/__pycache__/build_release.cpython-312.pyc and b/ops/devops/release/__pycache__/build_release.cpython-312.pyc differ diff --git a/ops/devops/release/__pycache__/verify_release.cpython-312.pyc b/ops/devops/release/__pycache__/verify_release.cpython-312.pyc index 832dfb120..7a0f262d6 100644 Binary files a/ops/devops/release/__pycache__/verify_release.cpython-312.pyc and b/ops/devops/release/__pycache__/verify_release.cpython-312.pyc differ diff --git a/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc b/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc index 3d7e3fd9a..99f43b556 100644 Binary files a/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc and b/ops/offline-kit/__pycache__/build_offline_kit.cpython-312.pyc differ diff --git a/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc b/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc new file mode 100644 index 000000000..b9097892d Binary files /dev/null and b/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc differ diff --git a/ops/offline-kit/build_offline_kit.py b/ops/offline-kit/build_offline_kit.py index 53b63a423..a37a5eb87 100644 --- a/ops/offline-kit/build_offline_kit.py +++ b/ops/offline-kit/build_offline_kit.py @@ -184,6 +184,45 @@ def copy_plugins_and_assets(staging_dir: Path) -> None: copy_if_exists(REPO_ROOT / "docs" / "24_OFFLINE_KIT.md", docs_dir / "24_OFFLINE_KIT.md") copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-collector.md", docs_dir / "telemetry-collector.md") copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-storage.md", docs_dir / "telemetry-storage.md") + copy_if_exists(REPO_ROOT / "docs" / "airgap" / "mirror-bundles.md", docs_dir / "mirror-bundles.md") + + +def copy_cli_and_taskrunner_assets(release_dir: Path, staging_dir: Path) -> None: + """Bundle CLI binaries, task pack docs, and Task Runner samples when available.""" + cli_src = release_dir / "cli" + if cli_src.exists(): + copy_if_exists(cli_src, staging_dir / "cli") + + taskrunner_bootstrap = staging_dir / "bootstrap" / "task-runner" + taskrunner_bootstrap.mkdir(parents=True, exist_ok=True) + copy_if_exists(REPO_ROOT / "etc" / "task-runner.yaml.sample", taskrunner_bootstrap / "task-runner.yaml.sample") + + docs_dir = staging_dir / "docs" + copy_if_exists(REPO_ROOT / "docs" / "task-packs", docs_dir / "task-packs") + copy_if_exists(REPO_ROOT / "docs" / "modules" / "taskrunner", docs_dir / "modules" / "taskrunner") + + +def copy_orchestrator_assets(release_dir: Path, staging_dir: Path) -> None: + """Copy orchestrator service, worker SDK, postgres snapshot, and dashboards when present.""" + mapping = { + release_dir / "orchestrator" / "service": staging_dir / "orchestrator" / "service", + release_dir / "orchestrator" / "worker-sdk": staging_dir / "orchestrator" / "worker-sdk", + release_dir / "orchestrator" / "postgres": staging_dir / "orchestrator" / "postgres", + release_dir / "orchestrator" / "dashboards": staging_dir / "orchestrator" / "dashboards", + } + for src, dest in mapping.items(): + copy_if_exists(src, dest) + + +def copy_export_and_notifier_assets(release_dir: Path, staging_dir: Path) -> None: + """Copy Export Center and Notifier offline bundles and tooling when present.""" + copy_if_exists(release_dir / "export-center", staging_dir / "export-center") + copy_if_exists(release_dir / "notifier", staging_dir / "notifier") + + +def copy_surface_secrets(release_dir: Path, staging_dir: Path) -> None: + """Include Surface.Secrets bundles and manifests if present.""" + copy_if_exists(release_dir / "surface-secrets", staging_dir / "surface-secrets") def copy_bootstrap_configs(staging_dir: Path) -> None: @@ -267,7 +306,21 @@ def scan_files(staging_dir: Path, exclude: Optional[set[str]] = None) -> list[Or ) ) ) - return entries + return entries + + +def copy_container_bundles(release_dir: Path, staging_dir: Path) -> None: + """Copy container air-gap bundles if present in the release directory.""" + candidates = [release_dir / "containers", release_dir / "images"] + target_dir = staging_dir / "containers" + for root in candidates: + if not root.exists(): + continue + for bundle in sorted(root.glob("**/*")): + if bundle.is_file() and bundle.suffix in {".gz", ".tar", ".tgz"}: + target_path = target_dir / bundle.relative_to(root) + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bundle, target_path) def write_offline_manifest( @@ -372,11 +425,16 @@ def build_offline_kit(args: argparse.Namespace) -> MutableMapping[str, Any]: if isinstance(checksums, Mapping): release_manifest_sha = checksums.get("sha256") - copy_release_manifests(release_dir, staging_dir) + copy_release_manifests(release_dir, staging_dir) copy_component_artifacts(manifest_data, release_dir, staging_dir) copy_collections(manifest_data, release_dir, staging_dir) copy_plugins_and_assets(staging_dir) copy_bootstrap_configs(staging_dir) + copy_cli_and_taskrunner_assets(release_dir, staging_dir) + copy_container_bundles(release_dir, staging_dir) + copy_orchestrator_assets(release_dir, staging_dir) + copy_export_and_notifier_assets(release_dir, staging_dir) + copy_surface_secrets(release_dir, staging_dir) copy_third_party_licenses(staging_dir) package_telemetry_bundle(staging_dir) diff --git a/ops/offline-kit/test_build_offline_kit.py b/ops/offline-kit/test_build_offline_kit.py index a1daa458f..253f3bdb3 100644 --- a/ops/offline-kit/test_build_offline_kit.py +++ b/ops/offline-kit/test_build_offline_kit.py @@ -40,8 +40,42 @@ class OfflineKitBuilderTests(unittest.TestCase): json.dump(payload, handle, indent=2) handle.write("\n") - def _create_sample_release(self) -> None: - self.release_dir.mkdir(parents=True, exist_ok=True) + def _create_sample_release(self) -> None: + self.release_dir.mkdir(parents=True, exist_ok=True) + + cli_archive = self.release_dir / "cli" / "stellaops-cli-linux-x64.tar.gz" + cli_archive.parent.mkdir(parents=True, exist_ok=True) + cli_archive.write_bytes(b"cli-bytes") + compute_sha256(cli_archive) + + container_bundle = self.release_dir / "containers" / "stellaops-containers.tar.gz" + container_bundle.parent.mkdir(parents=True, exist_ok=True) + container_bundle.write_bytes(b"container-bundle") + compute_sha256(container_bundle) + + orchestrator_service = self.release_dir / "orchestrator" / "service" / "orchestrator-service.tar.gz" + orchestrator_service.parent.mkdir(parents=True, exist_ok=True) + orchestrator_service.write_bytes(b"orch-service") + compute_sha256(orchestrator_service) + + orchestrator_dash = self.release_dir / "orchestrator" / "dashboards" / "dash.json" + orchestrator_dash.parent.mkdir(parents=True, exist_ok=True) + orchestrator_dash.write_text("{}\n", encoding="utf-8") + + export_bundle = self.release_dir / "export-center" / "export-offline-bundle.tar.gz" + export_bundle.parent.mkdir(parents=True, exist_ok=True) + export_bundle.write_bytes(b"export") + compute_sha256(export_bundle) + + notifier_pack = self.release_dir / "notifier" / "notifier-offline-pack.tar.gz" + notifier_pack.parent.mkdir(parents=True, exist_ok=True) + notifier_pack.write_bytes(b"notifier") + compute_sha256(notifier_pack) + + secrets_bundle = self.release_dir / "surface-secrets" / "secrets-bundle.tar.gz" + secrets_bundle.parent.mkdir(parents=True, exist_ok=True) + secrets_bundle.write_bytes(b"secrets") + compute_sha256(secrets_bundle) sbom_path = self.release_dir / "artifacts/sboms/sample.cyclonedx.json" sbom_path.parent.mkdir(parents=True, exist_ok=True) @@ -114,10 +148,10 @@ class OfflineKitBuilderTests(unittest.TestCase): encoding="utf-8", ) - manifest = OrderedDict( - ( - ( - "release", + manifest = OrderedDict( + ( + ( + "release", OrderedDict( ( ("version", "1.0.0"), @@ -216,9 +250,9 @@ class OfflineKitBuilderTests(unittest.TestCase): ), ) ) - write_manifest(manifest, self.release_dir) - - def test_build_offline_kit(self) -> None: + write_manifest(manifest, self.release_dir) + + def test_build_offline_kit(self) -> None: args = argparse.Namespace( version="2025.10.0", channel="edge", @@ -242,10 +276,34 @@ class OfflineKitBuilderTests(unittest.TestCase): self.assertTrue((bootstrap_notify / "notify.yaml").exists()) self.assertTrue((bootstrap_notify / "notify-web.secret.example").exists()) + taskrunner_bootstrap = self.staging_dir / "bootstrap" / "task-runner" + self.assertTrue((taskrunner_bootstrap / "task-runner.yaml.sample").exists()) + + docs_taskpacks = self.staging_dir / "docs" / "task-packs" + self.assertTrue(docs_taskpacks.exists()) + self.assertTrue((self.staging_dir / "docs" / "mirror-bundles.md").exists()) + + containers_dir = self.staging_dir / "containers" + self.assertTrue((containers_dir / "stellaops-containers.tar.gz").exists()) + + orchestrator_dir = self.staging_dir / "orchestrator" + self.assertTrue((orchestrator_dir / "service" / "orchestrator-service.tar.gz").exists()) + self.assertTrue((orchestrator_dir / "dashboards" / "dash.json").exists()) + + export_dir = self.staging_dir / "export-center" + self.assertTrue((export_dir / "export-offline-bundle.tar.gz").exists()) + + notifier_dir = self.staging_dir / "notifier" + self.assertTrue((notifier_dir / "notifier-offline-pack.tar.gz").exists()) + + secrets_dir = self.staging_dir / "surface-secrets" + self.assertTrue((secrets_dir / "secrets-bundle.tar.gz").exists()) + with offline_manifest.open("r", encoding="utf-8") as handle: manifest_data = json.load(handle) artifacts = manifest_data["artifacts"] self.assertTrue(any(item["name"].startswith("sboms/") for item in artifacts)) + self.assertTrue(any(item["name"].startswith("cli/") for item in artifacts)) metadata_path = Path(result["metadataPath"]) data = json.loads(metadata_path.read_text(encoding="utf-8")) @@ -258,6 +316,11 @@ class OfflineKitBuilderTests(unittest.TestCase): self.assertTrue(any(name.startswith("sboms/sample-") for name in members)) self.assertIn("bootstrap/notify/notify.yaml", members) self.assertIn("bootstrap/notify/notify-web.secret.example", members) + self.assertIn("containers/stellaops-containers.tar.gz", members) + self.assertIn("orchestrator/service/orchestrator-service.tar.gz", members) + self.assertIn("export-center/export-offline-bundle.tar.gz", members) + self.assertIn("notifier/notifier-offline-pack.tar.gz", members) + self.assertIn("surface-secrets/secrets-bundle.tar.gz", members) if __name__ == "__main__": diff --git a/out/telemetry/telemetry-offline-bundle.tar.gz b/out/telemetry/telemetry-offline-bundle.tar.gz new file mode 100644 index 000000000..e000162b0 Binary files /dev/null and b/out/telemetry/telemetry-offline-bundle.tar.gz differ diff --git a/out/telemetry/telemetry-offline-bundle.tar.gz.sha256 b/out/telemetry/telemetry-offline-bundle.tar.gz.sha256 new file mode 100644 index 000000000..2c189c9e2 --- /dev/null +++ b/out/telemetry/telemetry-offline-bundle.tar.gz.sha256 @@ -0,0 +1 @@ +dc3938d79d4e0b9a77e92dc6660391f36230b8d16c9b24b7164b6a1e6723666b telemetry-offline-bundle.tar.gz diff --git a/src/Api/StellaOps.Api.OpenApi/tasks.md b/src/Api/StellaOps.Api.OpenApi/tasks.md index f6baf1d94..a10461b50 100644 --- a/src/Api/StellaOps.Api.OpenApi/tasks.md +++ b/src/Api/StellaOps.Api.OpenApi/tasks.md @@ -7,4 +7,4 @@ | OAS-62-001 | DOING | Populate request/response examples for top 50 endpoints, including standard error envelope. | | OAS-62-002 | TODO | Add custom lint rules enforcing pagination, idempotency headers, naming conventions, and example coverage. | | OAS-63-001 | TODO | Implement compatibility diff tooling comparing previous release specs; classify breaking vs additive changes. | -| OAS-63-002 | TODO | Add `/.well-known/openapi` discovery endpoint schema metadata (extensions, version info). | +| OAS-63-002 | DONE (2025-11-24) | Discovery endpoint metadata and schema extensions added; composed spec exports `/.well-known/openapi` entry. | diff --git a/src/Graph/StellaOps.Graph.Api/Program.cs b/src/Graph/StellaOps.Graph.Api/Program.cs index 96eaa0c29..ccda25197 100644 --- a/src/Graph/StellaOps.Graph.Api/Program.cs +++ b/src/Graph/StellaOps.Graph.Api/Program.cs @@ -60,7 +60,9 @@ app.MapPost("/graph/search", async (HttpContext context, GraphSearchRequest requ return Results.Empty; } - await foreach (var line in service.SearchAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.SearchAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -113,7 +115,9 @@ app.MapPost("/graph/query", async (HttpContext context, GraphQueryRequest reques return Results.Empty; } - await foreach (var line in service.QueryAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.QueryAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -166,7 +170,9 @@ app.MapPost("/graph/paths", async (HttpContext context, GraphPathRequest request return Results.Empty; } - await foreach (var line in service.FindPathsAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.FindPathsAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -219,7 +225,9 @@ app.MapPost("/graph/diff", async (HttpContext context, GraphDiffRequest request, return Results.Empty; } - await foreach (var line in service.DiffAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.DiffAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -272,7 +280,8 @@ app.MapPost("/graph/export", async (HttpContext context, GraphExportRequest requ return Results.Empty; } - var job = await service.StartExportAsync(tenant!, request, ct); + var tenantId = tenant!; + var job = await service.StartExportAsync(tenantId, request, ct); var manifest = new { jobId = job.JobId, diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs index c4a403067..f365c4013 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs @@ -8,14 +8,16 @@ namespace StellaOps.Graph.Api.Services; public sealed class InMemoryGraphDiffService : IGraphDiffService { private readonly InMemoryGraphRepository _repository; + private readonly IGraphMetrics _metrics; private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - public InMemoryGraphDiffService(InMemoryGraphRepository repository) + public InMemoryGraphDiffService(InMemoryGraphRepository repository, IGraphMetrics metrics) { _repository = repository; + _metrics = metrics; } public async IAsyncEnumerable DiffAsync(string tenant, GraphDiffRequest request, [EnumeratorCancellation] CancellationToken ct = default) @@ -26,6 +28,7 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService var edgeBudgetRemaining = budget.Edges ?? 10000; var budgetRemaining = tileBudgetLimit; var seq = 0; + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var snapA = _repository.GetSnapshot(tenant, request.SnapshotA); var snapB = _repository.GetSnapshot(tenant, request.SnapshotB); @@ -39,6 +42,8 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService Details = new { request.SnapshotA, request.SnapshotB } }; yield return JsonSerializer.Serialize(new TileEnvelope("error", seq++, error, Cost(tileBudgetLimit, budgetRemaining)), Options); + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/diff")); yield break; } @@ -49,15 +54,15 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService foreach (var added in nodesB.Values.Where(n => !nodesA.ContainsKey(n.Id)).OrderBy(n => n.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } - yield return JsonSerializer.Serialize(new TileEnvelope("node_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options); - } + if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); yield return tile!; yield break; } + yield return JsonSerializer.Serialize(new TileEnvelope("node_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options); + } foreach (var removed in nodesA.Values.Where(n => !nodesB.ContainsKey(n.Id)).OrderBy(n => n.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } - yield return JsonSerializer.Serialize(new TileEnvelope("node_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options); - } + if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); yield return tile!; yield break; } + yield return JsonSerializer.Serialize(new TileEnvelope("node_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options); + } foreach (var common in nodesA.Keys.Intersect(nodesB.Keys, StringComparer.Ordinal).OrderBy(k => k, StringComparer.Ordinal)) { @@ -65,7 +70,7 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService var b = nodesB[common]; if (!AttributesEqual(a.Attributes, b.Attributes)) { - if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } + if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); yield return tile!; yield break; } var diff = new DiffTile { EntityType = "node", @@ -82,13 +87,13 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService { foreach (var added in edgesB.Values.Where(e => !edgesA.ContainsKey(e.Id)).OrderBy(e => e.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } + if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); yield return tile!; yield break; } yield return JsonSerializer.Serialize(new TileEnvelope("edge_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options); } foreach (var removed in edgesA.Values.Where(e => !edgesB.ContainsKey(e.Id)).OrderBy(e => e.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } + if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); yield return tile!; yield break; } yield return JsonSerializer.Serialize(new TileEnvelope("edge_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options); } @@ -98,12 +103,12 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService var b = edgesB[common]; if (!AttributesEqual(a.Attributes, b.Attributes)) { - if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } - var diff = new DiffTile - { - EntityType = "edge", - ChangeType = "changed", - Id = common, + if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); yield return tile!; yield break; } + var diff = new DiffTile + { + EntityType = "edge", + ChangeType = "changed", + Id = common, Before = a, After = b }; @@ -126,6 +131,9 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService yield return JsonSerializer.Serialize(new TileEnvelope("stats", seq++, stats, Cost(tileBudgetLimit, budgetRemaining)), Options); } + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/diff")); + await Task.CompletedTask; } diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs index 28dcb8231..52e000c4f 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs @@ -9,15 +9,17 @@ public sealed class InMemoryGraphPathService : IGraphPathService { private readonly InMemoryGraphRepository _repository; private readonly IOverlayService _overlayService; + private readonly IGraphMetrics _metrics; private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - public InMemoryGraphPathService(InMemoryGraphRepository repository, IOverlayService overlayService) + public InMemoryGraphPathService(InMemoryGraphRepository repository, IOverlayService overlayService, IGraphMetrics metrics) { _repository = repository; _overlayService = overlayService; + _metrics = metrics; } public async IAsyncEnumerable FindPathsAsync(string tenant, GraphPathRequest request, [EnumeratorCancellation] CancellationToken ct = default) @@ -29,6 +31,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService var edgeBudgetRemaining = budget.Edges ?? 10000; var budgetRemaining = tileBudgetLimit; var seq = 0; + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var result = FindShortestPath(tenant, request, maxDepth); @@ -42,6 +45,8 @@ public sealed class InMemoryGraphPathService : IGraphPathService }; yield return JsonSerializer.Serialize(new TileEnvelope("error", seq++, error, Cost(tileBudgetLimit, budgetRemaining)), Options); + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/paths")); yield break; } @@ -58,6 +63,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService { if (budgetRemaining <= 0 || nodeBudgetRemaining <= 0) { + _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "tiles")); yield return BudgetExceeded(tileBudgetLimit, budgetRemaining, seq++); yield break; } @@ -75,6 +81,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService { if (budgetRemaining <= 0 || edgeBudgetRemaining <= 0) { + _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "tiles")); yield return BudgetExceeded(tileBudgetLimit, budgetRemaining, seq++); yield break; } @@ -93,6 +100,9 @@ public sealed class InMemoryGraphPathService : IGraphPathService yield return JsonSerializer.Serialize(new TileEnvelope("stats", seq++, stats, Cost(tileBudgetLimit, budgetRemaining)), Options); } + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/paths")); + await Task.CompletedTask; } diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs index 2b72a6f0c..6ffb5b902 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs @@ -49,21 +49,17 @@ public sealed class InMemoryGraphQueryService : IGraphQueryService var cursorOffset = CursorCodec.Decode(request.Cursor); var (nodes, edges) = _repository.QueryGraph(tenant, request); + if (nodes.Count > nodeBudgetLimit) + { + _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); + yield return BuildBudgetError(cacheKey, "nodes", nodeBudgetLimit, nodes.Count, edges.Count, budget); + yield break; + } + if (request.IncludeEdges && edges.Count > edgeBudgetLimit) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); - var error = new ErrorResponse - { - Error = "GRAPH_BUDGET_EXCEEDED", - Message = $"Query exceeded edge budget (edges>{edgeBudgetLimit}).", - Details = new { nodes = nodes.Count, edges = edges.Count, budget } - }; - var errorLine = JsonSerializer.Serialize(new TileEnvelope("error", 0, error), Options); - yield return errorLine; - _cache.Set(cacheKey, new[] { errorLine }, new MemoryCacheEntryOptions - { - AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(2) - }); + yield return BuildBudgetError(cacheKey, "edges", edgeBudgetLimit, nodes.Count, edges.Count, budget); yield break; } @@ -206,4 +202,20 @@ public sealed class InMemoryGraphQueryService : IGraphQueryService private static CostBudget Cost(int limit, int remainingBudget) => new(limit, remainingBudget - 1, limit - (remainingBudget - 1)); + + private string BuildBudgetError(string cacheKey, string exceeded, int budgetLimit, int nodesCount, int edgesCount, GraphQueryBudget budget) + { + var error = new ErrorResponse + { + Error = "GRAPH_BUDGET_EXCEEDED", + Message = $"Query exceeded {exceeded} budget ({exceeded}>{budgetLimit}).", + Details = new { nodes = nodesCount, edges = edgesCount, budget } + }; + var errorLine = JsonSerializer.Serialize(new TileEnvelope("error", 0, error), Options); + _cache.Set(cacheKey, new[] { errorLine }, new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(2) + }); + return errorLine; + } } diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs index 09a148a9c..ca52eecc0 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs @@ -44,7 +44,7 @@ namespace StellaOps.Graph.Api.Services; } // Always return a fresh copy so we can inject a single explain trace without polluting cache. - var overlays = new Dictionary(cachedBase, StringComparer.Ordinal); + var overlays = new Dictionary(cachedBase!, StringComparer.Ordinal); if (sampleExplain && !explainEmitted) { diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs index 587f4113a..469dd9a9f 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs @@ -11,7 +11,8 @@ public class DiffServiceTests public async Task DiffAsync_EmitsAddedRemovedChangedAndStats() { var repo = new InMemoryGraphRepository(); - var service = new InMemoryGraphDiffService(repo); + var metrics = new GraphMetrics(); + var service = new InMemoryGraphDiffService(repo, metrics); var request = new GraphDiffRequest { @@ -37,7 +38,8 @@ public class DiffServiceTests public async Task DiffAsync_WhenSnapshotMissing_ReturnsError() { var repo = new InMemoryGraphRepository(); - var service = new InMemoryGraphDiffService(repo); + var metrics = new GraphMetrics(); + var service = new InMemoryGraphDiffService(repo, metrics); var request = new GraphDiffRequest { diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs index 6f6c1a880..72e789fab 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs @@ -13,8 +13,9 @@ public class PathServiceTests { var repo = new InMemoryGraphRepository(); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphPathService(repo, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphPathService(repo, overlays, metrics); var request = new GraphPathRequest { @@ -39,8 +40,9 @@ public class PathServiceTests { var repo = new InMemoryGraphRepository(); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphPathService(repo, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphPathService(repo, overlays, metrics); var request = new GraphPathRequest { diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs index 419fae382..d1dcee534 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs @@ -107,8 +107,9 @@ namespace StellaOps.Graph.Api.Tests; private static InMemoryGraphQueryService CreateService(InMemoryGraphRepository? repository = null) { - var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - return new InMemoryGraphQueryService(repository ?? new InMemoryGraphRepository(), cache, overlays); + var cache = new MemoryCache(new MemoryCacheOptions()); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + return new InMemoryGraphQueryService(repository ?? new InMemoryGraphRepository(), cache, overlays, metrics); } } diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs index ddc9bf3a1..464aa2ba8 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs @@ -121,8 +121,9 @@ public class SearchServiceTests }, Array.Empty()); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphQueryService(repo, cache, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphQueryService(repo, cache, overlays, metrics); var request = new GraphQueryRequest { Kinds = new[] { "component" }, @@ -154,8 +155,9 @@ public class SearchServiceTests }); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphQueryService(repo, cache, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphQueryService(repo, cache, overlays, metrics); var request = new GraphQueryRequest { Kinds = new[] { "component" }, diff --git a/src/Policy/StellaOps.Policy.min.slnf b/src/Policy/StellaOps.Policy.min.slnf new file mode 100644 index 000000000..9c0b4dbfa --- /dev/null +++ b/src/Policy/StellaOps.Policy.min.slnf @@ -0,0 +1,12 @@ +{ + "solution": { + "path": "StellaOps.Policy.sln", + "projects": [ + "__Libraries/StellaOps.Policy/StellaOps.Policy.csproj", + "__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj", + "../__Libraries/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj", + "../__Libraries/StellaOps.Configuration/StellaOps.Configuration.csproj", + "../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" + ] + } +} diff --git a/src/Policy/StellaOps.Policy.tests.slnf b/src/Policy/StellaOps.Policy.tests.slnf new file mode 100644 index 000000000..1a7f46a2d --- /dev/null +++ b/src/Policy/StellaOps.Policy.tests.slnf @@ -0,0 +1,9 @@ +{ + "solution": { + "path": "StellaOps.Policy.sln", + "projects": [ + "__Libraries/StellaOps.Policy/StellaOps.Policy.csproj", + "__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj" + ] + } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs index 7709313b5..5ef09dac2 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs @@ -10,6 +10,7 @@ using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; using StellaOps.Scanner.Analyzers.Lang; using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Entropy; using StellaOps.Scanner.EntryTrace; using StellaOps.Scanner.EntryTrace.Serialization; using StellaOps.Scanner.Surface.Env; @@ -185,6 +186,41 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor RegisterArtifact: true)); } + if (context.Analysis.TryGet(ScanAnalysisKeys.EntropyReport, out var entropyReport) && entropyReport is not null) + { + var json = JsonSerializer.Serialize(entropyReport, JsonOptions); + payloads.Add(new SurfaceManifestPayload( + ArtifactDocumentType.SurfaceObservation, + ArtifactDocumentFormat.ObservationJson, + Kind: "entropy.report", + MediaType: "application/json", + Content: Encoding.UTF8.GetBytes(json), + View: "entropy", + Metadata: new Dictionary + { + ["imageDigest"] = entropyReport.ImageDigest, + ["layerDigest"] = entropyReport.LayerDigest, + ["opaqueRatio"] = entropyReport.ImageOpaqueRatio.ToString("0.####", CultureInfoInvariant) + })); + } + + if (context.Analysis.TryGet(ScanAnalysisKeys.EntropyLayerSummary, out var entropySummary) && entropySummary is not null) + { + var json = JsonSerializer.Serialize(entropySummary, JsonOptions); + payloads.Add(new SurfaceManifestPayload( + ArtifactDocumentType.SurfaceObservation, + ArtifactDocumentFormat.ObservationJson, + Kind: "entropy.layer-summary", + MediaType: "application/json", + Content: Encoding.UTF8.GetBytes(json), + View: "entropy", + Metadata: new Dictionary + { + ["layerDigest"] = entropySummary.LayerDigest, + ["opaqueRatio"] = entropySummary.OpaqueRatio.ToString("0.####", CultureInfoInvariant) + })); + } + return payloads; } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs index 0e73560ca..8ff2ca248 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs @@ -28,4 +28,8 @@ public static class ScanAnalysisKeys public const string ReachabilityUnionGraph = "analysis.reachability.union.graph"; public const string ReachabilityUnionCas = "analysis.reachability.union.cas"; + + public const string FileEntries = "analysis.files.entries"; + public const string EntropyReport = "analysis.entropy.report"; + public const string EntropyLayerSummary = "analysis.entropy.layer.summary"; } diff --git a/src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs b/src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs new file mode 100644 index 000000000..789e45ae6 --- /dev/null +++ b/src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs @@ -0,0 +1,71 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Zastava.Observer.Configuration; +using StellaOps.Zastava.Observer.Runtime; + +namespace StellaOps.Zastava.Observer.Backend; + +internal interface IRuntimeFactsClient +{ + Task PublishAsync(RuntimeFactsPublishRequest request, CancellationToken cancellationToken); +} + +internal sealed class RuntimeFactsClient : IRuntimeFactsClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + private readonly HttpClient httpClient; + private readonly IOptionsMonitor observerOptions; + private readonly ILogger logger; + + public RuntimeFactsClient( + HttpClient httpClient, + IOptionsMonitor observerOptions, + ILogger logger) + { + this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PublishAsync(RuntimeFactsPublishRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var reachability = observerOptions.CurrentValue.Reachability ?? new ReachabilityRuntimeOptions(); + var endpoint = reachability.Endpoint; + if (string.IsNullOrWhiteSpace(endpoint)) + { + throw new RuntimeFactsException("Reachability endpoint is not configured."); + } + + using var message = new HttpRequestMessage(HttpMethod.Post, endpoint); + + if (!string.IsNullOrWhiteSpace(reachability.AnalysisId)) + { + message.Headers.TryAddWithoutValidation("X-Analysis-Id", reachability.AnalysisId); + } + + var json = JsonSerializer.Serialize(request, SerializerOptions); + message.Content = new StringContent(json, Encoding.UTF8, "application/json"); + message.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using var response = await httpClient.SendAsync(message, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + return; + } + + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + logger.LogWarning("Runtime facts publish failed with status {Status}: {Body}", (int)response.StatusCode, body); + throw new RuntimeFactsException($"Runtime facts publish failed with status {(int)response.StatusCode}."); + } +} + diff --git a/src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs new file mode 100644 index 000000000..1fd5ffb6d --- /dev/null +++ b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Zastava.Observer.Configuration; + +/// +/// Configuration for emitting runtime reachability facts to Signals. +/// +public sealed class ReachabilityRuntimeOptions +{ + /// + /// Enables runtime reachability fact publishing when true. + /// + public bool Enabled { get; set; } + + /// + /// Signals endpoint that accepts runtime facts (JSON or NDJSON). + /// + [Required(AllowEmptyStrings = false)] + public string Endpoint { get; set; } = "https://signals.internal/signals/runtime-facts"; + + /// + /// Required callgraph identifier used to correlate runtime facts with static graphs. + /// + [Required(AllowEmptyStrings = false)] + public string CallgraphId { get; set; } = string.Empty; + + /// + /// Optional analysis identifier forwarded as X-Analysis-Id. + /// + public string? AnalysisId { get; set; } + + /// + /// Maximum number of facts sent in a single publish attempt. + /// + [Range(1, 5000)] + public int BatchSize { get; set; } = 1000; + + /// + /// Maximum delay (seconds) before flushing a partially filled batch. + /// + [Range(typeof(double), "0.1", "30")] + public double FlushIntervalSeconds { get; set; } = 2; + + /// + /// Optional subject fallback when image data is missing. + /// + public string? SubjectScanId { get; set; } + + public string? SubjectImageDigest { get; set; } + + public string? SubjectComponent { get; set; } + + public string? SubjectVersion { get; set; } +} + diff --git a/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs index 5d7f6af2c..839a24795 100644 --- a/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs +++ b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs @@ -7,8 +7,8 @@ namespace StellaOps.Zastava.Observer.Configuration; /// /// Observer-specific configuration applied on top of the shared runtime options. /// -public sealed class ZastavaObserverOptions -{ +public sealed class ZastavaObserverOptions +{ public const string SectionName = "zastava:observer"; private const string DefaultContainerdSocket = "unix:///run/containerd/containerd.sock"; @@ -126,6 +126,12 @@ public sealed class ZastavaObserverOptions /// [Range(1, 128)] public int MaxEntrypointArguments { get; set; } = 32; + + /// + /// Runtime reachability fact publishing configuration. + /// + [Required] + public ReachabilityRuntimeOptions Reachability { get; init; } = new(); } public sealed class ZastavaObserverBackendOptions diff --git a/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs b/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs index e57c577ca..f4baea0fa 100644 --- a/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs +++ b/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs @@ -87,6 +87,14 @@ public static class ObserverServiceCollectionExtensions client.Timeout = TimeSpan.FromSeconds(Math.Clamp(backend.RequestTimeoutSeconds, 1, 120)); }); + services.AddHttpClient() + .ConfigureHttpClient((provider, client) => + { + var optionsMonitor = provider.GetRequiredService>(); + var observer = optionsMonitor.CurrentValue; + client.Timeout = TimeSpan.FromSeconds(Math.Clamp(observer.Backend.RequestTimeoutSeconds, 1, 120)); + }); + services.TryAddEnumerable(ServiceDescriptor.Singleton, ObserverRuntimeOptionsPostConfigure>()); // Surface environment + cache/manifest/secrets wiring diff --git a/src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs b/src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs new file mode 100644 index 000000000..7457611cd --- /dev/null +++ b/src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs @@ -0,0 +1,357 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using System.Linq; +using System.Collections.Generic; +using StellaOps.Zastava.Core.Contracts; +using StellaOps.Zastava.Observer.Configuration; + +namespace StellaOps.Zastava.Observer.Runtime; + +internal static class RuntimeFactsBuilder +{ + public static RuntimeFactsPublishRequest? Build( + IReadOnlyCollection envelopes, + ReachabilityRuntimeOptions options) + { + ArgumentNullException.ThrowIfNull(envelopes); + ArgumentNullException.ThrowIfNull(options); + + if (!options.Enabled || envelopes.Count == 0) + { + return null; + } + + if (string.IsNullOrWhiteSpace(options.CallgraphId)) + { + return null; + } + + var facts = new List(envelopes.Count); + ReachabilitySubjectPayload? subject = null; + + foreach (var envelope in envelopes) + { + var fact = TryBuildFact(envelope); + if (fact is null) + { + continue; + } + + facts.Add(fact); + subject ??= BuildSubject(envelope, options); + } + + if (facts.Count == 0) + { + return null; + } + + subject ??= BuildFallbackSubject(options); + if (subject is null) + { + return null; + } + + facts.Sort(CompareFacts); + + return new RuntimeFactsPublishRequest + { + CallgraphId = options.CallgraphId.Trim(), + Subject = subject, + Events = facts + }; + } + + private static RuntimeFactEventPayload? TryBuildFact(RuntimeEventEnvelope envelope) + { + ArgumentNullException.ThrowIfNull(envelope); + + var evt = envelope.Event; + if (evt.Kind != RuntimeEventKind.ContainerStart || evt.Process is null) + { + return null; + } + + var symbolId = BuildSymbolId(evt.Process); + if (string.IsNullOrWhiteSpace(symbolId)) + { + return null; + } + + var fact = new RuntimeFactEventPayload + { + SymbolId = symbolId, + CodeId = Normalize(evt.Process.BuildId), + BuildId = Normalize(evt.Process.BuildId), + LoaderBase = null, + Purl = null, + SymbolDigest = null, + HitCount = 1, + ObservedAt = evt.When, + ProcessId = evt.Process.Pid, + ProcessName = ResolveProcessName(evt.Process.Entrypoint), + ContainerId = evt.Workload.ContainerId, + Metadata = BuildMetadata(evt.Process) + }; + + return fact; + } + + private static string? BuildSymbolId(RuntimeProcess process) + { + if (!string.IsNullOrWhiteSpace(process.BuildId)) + { + return $"sym:binary:{process.BuildId.Trim().ToLowerInvariant()}"; + } + + var trace = process.EntryTrace?.FirstOrDefault(t => + !string.IsNullOrWhiteSpace(t.Target) || !string.IsNullOrWhiteSpace(t.File)); + + var seed = trace?.Target ?? trace?.File ?? process.Entrypoint.FirstOrDefault(); + if (string.IsNullOrWhiteSpace(seed)) + { + return null; + } + + var stable = ComputeStableFragment(seed); + return $"sym:shell:{stable}"; + } + + private static ReachabilitySubjectPayload? BuildSubject(RuntimeEventEnvelope envelope, ReachabilityRuntimeOptions options) + { + var workload = envelope.Event.Workload; + var imageRef = workload.ImageRef; + + var digest = ExtractImageDigest(imageRef); + var (component, version) = ExtractComponentAndVersion(imageRef); + + digest ??= Normalize(options.SubjectImageDigest); + component ??= Normalize(options.SubjectComponent); + version ??= Normalize(options.SubjectVersion); + + var scanId = Normalize(options.SubjectScanId); + + if (string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(scanId) + && (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version))) + { + return null; + } + + return new ReachabilitySubjectPayload + { + ScanId = scanId, + ImageDigest = digest, + Component = component, + Version = version + }; + } + + private static ReachabilitySubjectPayload? BuildFallbackSubject(ReachabilityRuntimeOptions options) + { + var digest = Normalize(options.SubjectImageDigest); + var component = Normalize(options.SubjectComponent); + var version = Normalize(options.SubjectVersion); + var scanId = Normalize(options.SubjectScanId); + + if (string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(scanId) + && (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version))) + { + return null; + } + + return new ReachabilitySubjectPayload + { + ScanId = scanId, + ImageDigest = digest, + Component = component, + Version = version + }; + } + + private static Dictionary? BuildMetadata(RuntimeProcess process) + { + if (process.EntryTrace is null || process.EntryTrace.Count == 0) + { + return null; + } + + var sb = new StringBuilder(); + foreach (var trace in process.EntryTrace) + { + if (trace is null) + { + continue; + } + + var op = string.IsNullOrWhiteSpace(trace.Op) ? "exec" : trace.Op; + var target = trace.Target ?? trace.File; + if (string.IsNullOrWhiteSpace(target)) + { + continue; + } + + if (sb.Length > 0) + { + sb.Append(" | "); + } + sb.Append(op).Append(':').Append(target); + } + + if (sb.Length == 0) + { + return null; + } + + return new Dictionary(StringComparer.Ordinal) + { + ["entryTrace"] = sb.ToString() + }; + } + + private static string? ExtractImageDigest(string? imageRef) + { + if (string.IsNullOrWhiteSpace(imageRef)) + { + return null; + } + + var digestStart = imageRef.IndexOf("sha256:", StringComparison.OrdinalIgnoreCase); + if (digestStart >= 0) + { + return imageRef[digestStart..].Trim(); + } + + var atIndex = imageRef.IndexOf('@'); + if (atIndex > 0 && atIndex + 1 < imageRef.Length) + { + return imageRef[(atIndex + 1)..].Trim(); + } + + return null; + } + + private static (string? Component, string? Version) ExtractComponentAndVersion(string? imageRef) + { + if (string.IsNullOrWhiteSpace(imageRef)) + { + return (null, null); + } + + var lastSlash = imageRef.LastIndexOf('/'); + var lastColon = imageRef.LastIndexOf(':'); + + if (lastColon < 0 || lastColon < lastSlash) + { + return (null, null); + } + + var component = imageRef[(lastSlash + 1)..lastColon]; + var version = imageRef[(lastColon + 1)..]; + + if (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version)) + { + return (null, null); + } + + return (component.Trim(), version.Trim()); + } + + private static string? ResolveProcessName(IReadOnlyList? entrypoint) + { + if (entrypoint is null || entrypoint.Count == 0) + { + return null; + } + + var first = entrypoint[0]; + if (string.IsNullOrWhiteSpace(first)) + { + return null; + } + + var lastSlash = first.LastIndexOf('/'); + return lastSlash >= 0 ? first[(lastSlash + 1)..] : first; + } + + private static string ComputeStableFragment(string seed) + { + var normalized = seed.Trim().ToLowerInvariant(); + var bytes = Encoding.UTF8.GetBytes(normalized); + Span hash = stackalloc byte[32]; + _ = SHA256.TryHashData(bytes, hash, out _); + return Convert.ToHexString(hash[..16]).ToLowerInvariant(); + } + + private static int CompareFacts(RuntimeFactEventPayload left, RuntimeFactEventPayload right) + { + var timeComparison = Nullable.Compare(left.ObservedAt, right.ObservedAt); + if (timeComparison != 0) + { + return timeComparison; + } + + return string.Compare(left.SymbolId, right.SymbolId, StringComparison.Ordinal); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); +} + +internal sealed class RuntimeFactsPublishRequest +{ + public ReachabilitySubjectPayload Subject { get; set; } = new(); + + public string CallgraphId { get; set; } = string.Empty; + + public List Events { get; set; } = new(); +} + +internal sealed class ReachabilitySubjectPayload +{ + public string? ScanId { get; set; } + + public string? ImageDigest { get; set; } + + public string? Component { get; set; } + + public string? Version { get; set; } +} + +internal sealed class RuntimeFactEventPayload +{ + public string SymbolId { get; set; } = string.Empty; + + public string? CodeId { get; set; } + + public string? SymbolDigest { get; set; } + + public string? Purl { get; set; } + + public string? BuildId { get; set; } + + public string? LoaderBase { get; set; } + + public int? ProcessId { get; set; } + + public string? ProcessName { get; set; } + + public string? SocketAddress { get; set; } + + public string? ContainerId { get; set; } + + public string? EvidenceUri { get; set; } + + public int HitCount { get; set; } = 1; + + public DateTimeOffset? ObservedAt { get; set; } + + public Dictionary? Metadata { get; set; } +} + +internal sealed class RuntimeFactsException : Exception +{ + public RuntimeFactsException(string message, Exception? innerException = null) : base(message, innerException) + { + } +} diff --git a/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs b/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs index e681d3ca7..b8805526b 100644 --- a/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs +++ b/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs @@ -13,6 +13,7 @@ internal sealed class RuntimeEventDispatchService : BackgroundService { private readonly IRuntimeEventBuffer buffer; private readonly IRuntimeEventsClient eventsClient; + private readonly IRuntimeFactsClient runtimeFactsClient; private readonly IOptionsMonitor observerOptions; private readonly TimeProvider timeProvider; private readonly ILogger logger; @@ -20,12 +21,14 @@ internal sealed class RuntimeEventDispatchService : BackgroundService public RuntimeEventDispatchService( IRuntimeEventBuffer buffer, IRuntimeEventsClient eventsClient, + IRuntimeFactsClient runtimeFactsClient, IOptionsMonitor observerOptions, TimeProvider timeProvider, ILogger logger) { this.buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); this.eventsClient = eventsClient ?? throw new ArgumentNullException(nameof(eventsClient)); + this.runtimeFactsClient = runtimeFactsClient ?? throw new ArgumentNullException(nameof(runtimeFactsClient)); this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions)); this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -116,14 +119,17 @@ internal sealed class RuntimeEventDispatchService : BackgroundService return; } - var request = new RuntimeEventsIngestRequest - { - BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}", - Events = batch.Select(item => item.Envelope).ToArray() - }; - try { + var envelopes = batch.Select(item => item.Envelope).ToArray(); + var factsPublished = await TryPublishRuntimeFactsAsync(envelopes, cancellationToken).ConfigureAwait(false); + + var request = new RuntimeEventsIngestRequest + { + BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}", + Events = envelopes + }; + var result = await eventsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false); if (result.Success) { @@ -132,10 +138,11 @@ internal sealed class RuntimeEventDispatchService : BackgroundService await item.CompleteAsync().ConfigureAwait(false); } - logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}).", + logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}, runtimeFacts={FactsPublished}).", request.BatchId, result.Accepted, - result.Duplicates); + result.Duplicates, + factsPublished); } else if (result.RateLimited) { @@ -166,6 +173,38 @@ internal sealed class RuntimeEventDispatchService : BackgroundService } } + private async Task TryPublishRuntimeFactsAsync(RuntimeEventEnvelope[] envelopes, CancellationToken cancellationToken) + { + if (envelopes.Length == 0) + { + return false; + } + + var options = observerOptions.CurrentValue.Reachability; + var request = RuntimeFactsBuilder.Build(envelopes, options); + if (request is null) + { + return false; + } + + try + { + await runtimeFactsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false); + logger.LogDebug("Published {Count} runtime facts (callgraphId={CallgraphId}).", request.Events.Count, request.CallgraphId); + return true; + } + catch (RuntimeFactsException ex) when (!cancellationToken.IsCancellationRequested) + { + logger.LogWarning(ex, "Runtime facts publish failed; batch will be retried."); + throw; + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + logger.LogWarning(ex, "Runtime facts publish encountered an unexpected error; batch will be retried."); + throw; + } + } + private async Task RequeueBatchAsync(IEnumerable batch, CancellationToken cancellationToken) { foreach (var item in batch) diff --git a/src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs b/src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs new file mode 100644 index 000000000..f32f81896 --- /dev/null +++ b/src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using StellaOps.Zastava.Core.Contracts; +using StellaOps.Zastava.Observer.Configuration; +using StellaOps.Zastava.Observer.Runtime; +using Xunit; + +namespace StellaOps.Zastava.Observer.Tests.Runtime; + +public sealed class RuntimeFactsBuilderTests +{ + [Fact] + public void Build_UsesBuildIdAndDigest() + { + var options = new ReachabilityRuntimeOptions + { + Enabled = true, + CallgraphId = "cg-001" + }; + + var runtimeEvent = CreateRuntimeEvent( + imageRef: "ghcr.io/example/api@sha256:deadbeef", + buildId: "beadfeed", + when: DateTimeOffset.Parse("2025-11-26T12:00:00Z")); + + var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + var result = RuntimeFactsBuilder.Build(new[] { envelope }, options); + + Assert.NotNull(result); + Assert.Equal("cg-001", result!.CallgraphId); + Assert.Equal("sha256:deadbeef", result.Subject.ImageDigest); + Assert.Single(result.Events); + Assert.StartsWith("sym:binary:beadfeed", result.Events[0].SymbolId, StringComparison.Ordinal); + } + + [Fact] + public void Build_ParsesComponentAndVersion_WhenTagPresent() + { + var options = new ReachabilityRuntimeOptions + { + Enabled = true, + CallgraphId = "cg-002" + }; + + var runtimeEvent = CreateRuntimeEvent( + imageRef: "registry.local/team/web:1.2.3", + buildId: null, + when: DateTimeOffset.Parse("2025-11-26T12:01:00Z")); + + var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + var result = RuntimeFactsBuilder.Build(new[] { envelope }, options); + + Assert.NotNull(result); + Assert.Equal("web", result!.Subject.Component); + Assert.Equal("1.2.3", result.Subject.Version); + Assert.Single(result.Events); + Assert.StartsWith("sym:shell:", result.Events[0].SymbolId, StringComparison.Ordinal); + } + + private static RuntimeEvent CreateRuntimeEvent(string imageRef, string? buildId, DateTimeOffset when) + { + var process = new RuntimeProcess + { + Pid = 1234, + Entrypoint = new[] { "/entrypoint.sh", "-c", "echo hi" }, + EntryTrace = new[] { new RuntimeEntryTrace { File = "/entrypoint.sh", Op = "exec", Target = "/entrypoint.sh" } }, + BuildId = buildId + }; + + return new RuntimeEvent + { + EventId = "evt-1", + When = when, + Kind = RuntimeEventKind.ContainerStart, + Tenant = "tenant-a", + Node = "node-a", + Runtime = new RuntimeEngine { Engine = "containerd", Version = "1.7" }, + Workload = new RuntimeWorkload + { + Platform = "kubernetes", + Namespace = "default", + Pod = "pod-a", + Container = "api", + ContainerId = "containerd://abc", + ImageRef = imageRef + }, + Process = process, + LoadedLibraries = Array.Empty(), + Evidence = Array.Empty() + }; + } +} +