From 4831c7fcb0d8c8d9bdf01c106f8bcf27810f4fb0 Mon Sep 17 00:00:00 2001 From: StellaOps Bot Date: Wed, 26 Nov 2025 09:28:16 +0200 Subject: [PATCH] up --- docs/24_OFFLINE_KIT.md | 23 +- docs/api/graph.md | 123 ++++++ docs/api/vuln.md | 29 ++ docs/implplan/SPRINT_0207_0001_0001_graph.md | 4 +- ...PRINT_0321_0001_0001_docs_modules_graph.md | 6 +- ..._0001_reachability_runtime_static_union.md | 7 +- .../SPRINT_0513_0001_0001_provenance.md | 1 + docs/implplan/SPRINT_304_docs_tasks_md_iv.md | 11 +- docs/implplan/SPRINT_508_ops_offline_kit.md | 21 +- docs/implplan/tasks-all.md | 12 +- docs/migration/exception-governance.md | 64 ++++ docs/modules/cli/guides/graph-and-vuln.md | 82 ++++ docs/modules/graph/README.md | 15 + docs/modules/graph/architecture-index.md | 48 +++ .../__pycache__/build_release.cpython-312.pyc | Bin 57151 -> 57151 bytes .../verify_release.cpython-312.pyc | Bin 16403 -> 16403 bytes .../build_offline_kit.cpython-312.pyc | Bin 25407 -> 28663 bytes .../mirror_debug_store.cpython-312.pyc | Bin 0 -> 10973 bytes ops/offline-kit/build_offline_kit.py | 62 ++- ops/offline-kit/test_build_offline_kit.py | 81 +++- out/telemetry/telemetry-offline-bundle.tar.gz | Bin 0 -> 10988 bytes .../telemetry-offline-bundle.tar.gz.sha256 | 1 + src/Api/StellaOps.Api.OpenApi/tasks.md | 2 +- src/Graph/StellaOps.Graph.Api/Program.cs | 19 +- .../Services/InMemoryGraphDiffService.cs | 40 +- .../Services/InMemoryGraphPathService.cs | 12 +- .../Services/InMemoryGraphQueryService.cs | 36 +- .../Services/InMemoryOverlayService.cs | 2 +- .../DiffServiceTests.cs | 6 +- .../PathServiceTests.cs | 10 +- .../QueryServiceTests.cs | 7 +- .../SearchServiceTests.cs | 10 +- src/Policy/StellaOps.Policy.min.slnf | 12 + src/Policy/StellaOps.Policy.tests.slnf | 9 + .../Surface/SurfaceManifestStageExecutor.cs | 36 ++ .../Contracts/ScanAnalysisKeys.cs | 4 + .../Backend/RuntimeFactsClient.cs | 71 ++++ .../ReachabilityRuntimeOptions.cs | 55 +++ .../Configuration/ZastavaObserverOptions.cs | 10 +- .../ObserverServiceCollectionExtensions.cs | 8 + .../Runtime/RuntimeFactsBuilder.cs | 357 ++++++++++++++++++ .../Worker/RuntimeEventDispatchService.cs | 55 ++- .../Runtime/RuntimeFactsBuilderTests.cs | 93 +++++ 43 files changed, 1347 insertions(+), 97 deletions(-) create mode 100644 docs/api/graph.md create mode 100644 docs/api/vuln.md create mode 100644 docs/migration/exception-governance.md create mode 100644 docs/modules/cli/guides/graph-and-vuln.md create mode 100644 docs/modules/graph/architecture-index.md create mode 100644 ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc create mode 100644 out/telemetry/telemetry-offline-bundle.tar.gz create mode 100644 out/telemetry/telemetry-offline-bundle.tar.gz.sha256 create mode 100644 src/Policy/StellaOps.Policy.min.slnf create mode 100644 src/Policy/StellaOps.Policy.tests.slnf create mode 100644 src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs create mode 100644 src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs create mode 100644 src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs create mode 100644 src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md index 5f6519cba..73465303f 100755 --- a/docs/24_OFFLINE_KIT.md +++ b/docs/24_OFFLINE_KIT.md @@ -15,11 +15,15 @@ completely isolated network: | **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU | | **Container images** | `stella-ops`, *Zastava* sidecar, `advisory-ai-web`, and `advisory-ai-worker` (x86‑64 & arm64) | | **Provenance** | Cosign signature, SPDX 2.3 SBOM, in‑toto SLSA attestation | -| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | -| **Delta patches** | Daily diff bundles keep size \< 350 MB | +| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | +| **Delta patches** | Daily diff bundles keep size \< 350 MB | | **Scanner plug-ins** | OS analyzers plus the Node.js, Go, .NET, Python, and Rust language analyzers packaged under `plugins/scanner/analyzers/**` with manifests so Workers load deterministically offline. | -| **Debug store** | `.debug` artefacts laid out under `debug/.build-id//.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. | -| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. | +| **Debug store** | `.debug` artefacts laid out under `debug/.build-id//.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. | +| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. | +| **CLI + Task Packs** | `cli/` binaries from `release/cli`, Task Runner bootstrap (`bootstrap/task-runner/task-runner.yaml.sample`), and task-pack docs under `docs/task-packs/**` + `docs/modules/taskrunner/**`. | +| **Orchestrator/Export/Notifier kits** | Orchestrator service, worker SDK, Postgres snapshot, dashboards (`orchestrator/**`), Export Center bundles (`export-center/**`), Notifier offline packs (`notifier/**`). | +| **Container air-gap bundles** | Any tar/tgz under `containers/` or `images/` (mirrored registries) plus `docs/airgap/mirror-bundles.md`. | +| **Surface.Secrets** | Encrypted secrets bundles and manifests (`surface-secrets/**`) for sealed-mode bootstrap. | **RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an air‑gapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache. @@ -46,8 +50,15 @@ The helper copies `debug/.build-id/**`, validates `debug/debug-manifest.json` ag ## 0.1 · Automated packaging -The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`. -It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set. +The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`. +It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set. + +What it picks up automatically (if present under `--release-dir`): +- `cli/**` → CLI binaries and installers. +- `containers/**` or `images/**` → air-gap container bundles. +- `orchestrator/{service,worker-sdk,postgres,dashboards}/**`. +- `export-center/**`, `notifier/**`, `surface-secrets/**`. +- Docs: `docs/task-packs/**`, `docs/modules/taskrunner/**`, `docs/airgap/mirror-bundles.md`. ```bash python ops/offline-kit/build_offline_kit.py \ diff --git a/docs/api/graph.md b/docs/api/graph.md new file mode 100644 index 000000000..660aa6bf0 --- /dev/null +++ b/docs/api/graph.md @@ -0,0 +1,123 @@ +# Graph API + +Status: Draft (2025-11-26) — aligns with Sprint 0207 Graph API implementation (in-memory seed; RBAC + budgets enforced). + +Base URL: `/api/graph` (examples use relative paths). + +## Common headers +- `X-Stella-Tenant` (required) +- `Authorization: Bearer ` (required) +- `X-Stella-Scopes`: space/comma/semicolon separated scopes + - `/graph/search`: `graph:read` or `graph:query` + - `/graph/query|paths|diff`: `graph:query` + - `/graph/export`: `graph:export` +Content-Type for requests: `application/json` +Streaming responses: `application/x-ndjson` + +## POST /graph/search +Returns NDJSON stream of tiles (`node`, optional `cursor`). + +Body: +```json +{ + "kinds": ["component", "artifact"], + "query": "pkg:npm/", + "filters": { "ecosystem": "npm" }, + "limit": 50, + "cursor": "opaque" +} +``` +Errors: +- 400 `GRAPH_VALIDATION_FAILED` (missing kinds/query/filters) +- 401 missing auth; 403 missing scopes + +## POST /graph/query +Streams nodes, edges (optional), stats, cursor with cost metadata. + +Body: +```json +{ + "kinds": ["component"], + "query": "widget", + "filters": { "tenant": "acme" }, + "includeEdges": true, + "includeStats": true, + "includeOverlays": true, + "limit": 100, + "cursor": "opaque", + "budget": { "tiles": 6000, "nodes": 5000, "edges": 10000 } +} +``` +Error tile if edge budget exceeded: `{ "type":"error","data":{"error":"GRAPH_BUDGET_EXCEEDED",...}}` + +## POST /graph/paths +Finds paths up to depth 6 between sources/targets. + +Body: +```json +{ + "sources": ["gn:acme:component:one"], + "targets": ["gn:acme:component:two"], + "kinds": ["depends_on"], + "maxDepth": 4, + "includeOverlays": false, + "budget": { "tiles": 2000, "nodes": 1500, "edges": 3000 } +} +``` +Response: NDJSON tiles (`node`, `edge`, `stats`, `cursor`). + +## POST /graph/diff +Diff two snapshots. + +Body: +```json +{ + "snapshotA": "snapA", + "snapshotB": "snapB", + "includeEdges": true, + "includeStats": true, + "budget": { "tiles": 2000 } +} +``` +Response tiles: `node`/`edge` with change types, `stats`, optional `cursor`. + +## POST /graph/export +Kicks off an in-memory export job and returns manifest. + +Body: +```json +{ + "format": "ndjson", // ndjson|csv|graphml|png|svg + "includeEdges": true, + "snapshotId": "snapA", // optional + "kinds": ["component"], // optional + "query": "pkg:", // optional + "filters": { "ecosystem": "npm" } +} +``` +Response: +```json +{ + "jobId": "job-123", + "status": "completed", + "format": "ndjson", + "sha256": "...", + "size": 1234, + "downloadUrl": "/graph/export/job-123", + "completedAt": "2025-11-26T00:00:00Z" +} +``` +Download: `GET /graph/export/{jobId}` (returns file, headers include `X-Content-SHA256`). + +## Health +`GET /healthz` → 200 when service is ready; no auth/scopes required. + +## Error envelope +```json +{ "error": "GRAPH_VALIDATION_FAILED", "message": "details", "requestId": "optional" } +``` + +## Notes +- All timestamps UTC ISO-8601. +- Ordering is deterministic; cursors are opaque base64 offsets. +- Overlays use `policy.overlay.v1` and `openvex.v1` payloads. Budgeted tiles reserve room for cursors when `hasMore` is true. diff --git a/docs/api/vuln.md b/docs/api/vuln.md new file mode 100644 index 000000000..703097fbd --- /dev/null +++ b/docs/api/vuln.md @@ -0,0 +1,29 @@ +# Vulnerability API (placeholder) + +Status: Draft (2025-11-26) — awaiting Vuln Explorer v1 surface. This doc reserves the path and headers to align with upcoming releases. + +## Base URL +`/api/vuln` (subject to final routing via API gateway). + +## Common headers +- `X-Stella-Tenant` (required) +- `Authorization: Bearer ` +- `X-Stella-Scopes`: expect `vuln:read` (TBD) and/or `graph:read` when graph-backed queries are invoked. +- `Content-Type: application/json` + +## Planned endpoints (subject to change) +- `POST /vuln/search` — filter vulnerabilities by component (purl/digest), advisory id, status, exploitability (OpenVEX). +- `POST /vuln/impact` — compute impacted assets using Graph overlays; may proxy to Graph API internally. +- `GET /vuln/{id}` — details with references, VEX status, nearest safe version. +- `GET /vuln/{id}/evidence` — raw evidence (SBOM snapshot refs, observations). +- `GET /vuln/kev` — Known Exploited Vulnerabilities view (cached). + +## Error envelope +Follows Graph/Platform standard: +```json +{ "error": "VULN_VALIDATION_FAILED", "message": "details", "requestId": "optional" } +``` + +## Notes +- This placeholder will be updated once Vuln Explorer API is finalized. Keep gateway clients tolerant to minor shape changes until status flips to READY. +- For current graph-backed queries, use `/graph/search` or `/graph/query` (see `docs/api/graph.md`). diff --git a/docs/implplan/SPRINT_0207_0001_0001_graph.md b/docs/implplan/SPRINT_0207_0001_0001_graph.md index 8d1787e3e..007ee8f25 100644 --- a/docs/implplan/SPRINT_0207_0001_0001_graph.md +++ b/docs/implplan/SPRINT_0207_0001_0001_graph.md @@ -31,9 +31,9 @@ | 6 | GRAPH-API-28-006 | DONE (2025-11-26) | GRAPH-API-28-005; POLICY-ENGINE-30-001..003 contracts | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Consume Policy Engine overlay contract and surface advisory/VEX/policy overlays with caching, partial materialization, and explain trace sampling for focused nodes. | | 7 | GRAPH-API-28-007 | DONE (2025-11-26) | GRAPH-API-28-006 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Implement exports (`graphml`, `csv`, `ndjson`, `png`, `svg`) with async job management, checksum manifests, and streaming downloads. | | 8 | GRAPH-API-28-008 | DONE (2025-11-26) | GRAPH-API-28-007 | Graph API + Authority Guilds (`src/Graph/StellaOps.Graph.Api`) | Integrate RBAC scopes (`graph:read`, `graph:query`, `graph:export`), tenant headers, audit logging, and rate limiting. | -| 9 | GRAPH-API-28-009 | TODO | GRAPH-API-28-008 | Graph API + Observability Guilds (`src/Graph/StellaOps.Graph.Api`) | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. | +| 9 | GRAPH-API-28-009 | DONE (2025-11-26) | GRAPH-API-28-008 | Graph API + Observability Guilds (`src/Graph/StellaOps.Graph.Api`) | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. | | 10 | GRAPH-API-28-010 | DONE (2025-11-26) | GRAPH-API-28-009 | Graph API Guild · QA Guild (`src/Graph/StellaOps.Graph.Api`) | Build unit/integration/load tests with synthetic datasets (500k nodes/2M edges), fuzz query validation, verify determinism across runs. | -| 11 | GRAPH-API-28-011 | TODO | GRAPH-API-28-010 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. | +| 11 | GRAPH-API-28-011 | DONE (2025-11-26) | GRAPH-API-28-010 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. | | 12 | GRAPH-INDEX-28-011 | DONE (2025-11-04) | Downstream consumption by API once overlays ready | Graph Indexer Guild (`src/Graph/StellaOps.Graph.Indexer`) | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. | ## Wave Coordination diff --git a/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md b/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md index 0386a463b..196c878df 100644 --- a/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md +++ b/docs/implplan/SPRINT_0321_0001_0001_docs_modules_graph.md @@ -22,8 +22,8 @@ | --- | --- | --- | --- | --- | --- | | P1 | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | DONE (2025-11-22) | Due 2025-11-25 · Accountable: Ops Guild | Ops Guild | Waiting for next demo outputs to review dashboards/runbooks.

Document artefact/deliverable for GRAPH-OPS-0001 and publish location so downstream tasks can proceed. Prep artefact: `docs/modules/graph/prep/2025-11-20-ops-0001-prep.md`. | | 1 | GRAPH-ENG-0001 | DONE | Synced docs to Sprint 0141 rename on 2025-11-17 | Module Team | Keep module milestones in sync with `/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md` and related files; update references and note deltas. | -| 2 | GRAPH-DOCS-0002 | BLOCKED | Await DOCS-GRAPH-24-003 cross-links | Docs Guild | Add API/query doc cross-links once DOCS-GRAPH-24-003 lands. | -| 3 | GRAPH-OPS-0001 | TODO | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | Ops Guild | Review graph observability dashboards/runbooks after the next sprint demo; capture updates in runbooks. | +| 2 | GRAPH-DOCS-0002 | DONE (2025-11-26) | DOCS-GRAPH-24-003 delivered | Docs Guild | Add API/query doc cross-links once DOCS-GRAPH-24-003 lands. | +| 3 | GRAPH-OPS-0001 | DONE (2025-11-26) | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | Ops Guild | Review graph observability dashboards/runbooks after the next sprint demo; capture updates in runbooks. | ## Execution Log | Date (UTC) | Update | Owner | @@ -35,6 +35,8 @@ | 2025-11-17 | Normalised sprint to standard template; renamed from SPRINT_321_docs_modules_graph.md. | Docs | | 2025-11-22 | Marked all PREP tasks to DONE per directive; evidence to be verified. | Project Mgmt | | 2025-11-22 | PREP-GRAPH-OPS-0001 done; moved GRAPH-OPS-0001 to TODO pending next demo outputs. | Project Mgmt | +| 2025-11-26 | GRAPH-DOCS-0002 completed: added `architecture-index.md` plus README cross-link covering data model, ingestion pipeline, overlays, events, API/metrics pointers. | Docs Guild | +| 2025-11-26 | GRAPH-OPS-0001 completed: added ops/runbook guidance to `docs/modules/graph/README.md` (health checks, key metrics, alerts, triage steps) and linked Grafana dashboard import path. | Ops Guild | ## Decisions & Risks - Cross-links blocked on DOCS-GRAPH-24-003; track before marking GRAPH-DOCS-0002 done. diff --git a/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md b/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md index 5dfae5389..73905c8e5 100644 --- a/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md +++ b/docs/implplan/SPRINT_0400_0001_0001_reachability_runtime_static_union.md @@ -20,7 +20,7 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | ZASTAVA-REACH-201-001 | DOING (2025-11-26) | Need runtime symbol sampling design; align with GAP-ZAS-002 | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. | +| 1 | ZASTAVA-REACH-201-001 | DONE (2025-11-26) | Runtime facts emitter shipped in Observer | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. | | 9 | GAP-ZAS-002 | BLOCKED (2025-11-26) | Align with task 1; runtime NDJSON schema | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint 0401 lands ingestion. | | 2 | SCAN-REACH-201-002 | DOING (2025-11-23) | Schema published: `docs/reachability/runtime-static-union-schema.md` (v0.1). Implement emitters against CAS layout. | Scanner Worker Guild | Ship language-aware static lifters (JVM, .NET/Roslyn+IL, Go SSA, Node/Deno TS AST, Rust MIR, Swift SIL, shell/binary analyzers) in Scanner Worker; emit canonical SymbolIDs, CAS-stored graphs, and attach reachability tags to SBOM components. | | 3 | SIGNALS-REACH-201-003 | DONE (2025-11-25) | Consume schema `docs/reachability/runtime-static-union-schema.md`; wire ingestion + CAS storage. | Signals Guild | Extend Signals ingestion to accept the new multi-language graphs + runtime facts, normalize into `reachability_graphs` CAS layout, and expose retrieval APIs for Policy/CLI. | @@ -29,13 +29,14 @@ | 6 | DOCS-REACH-201-006 | DONE (2025-11-26) | Requires outputs from 1–5 | Docs Guild | Author the reachability doc set (`docs/reachability/reachability.md`, `callgraph-formats.md`, `runtime-facts.md`, CLI/UI appendices) plus update Zastava + Replay guides with the new evidence and operator workflows. | | 7 | QA-REACH-201-007 | DONE (2025-11-25) | Move fixtures + create evaluator harness | QA Guild | Integrate `reachbench-2025-expanded` fixture pack under `tests/reachability/fixtures/`, add evaluator harness tests that validate reachable vs unreachable cases, and wire CI guidance for deterministic runs. | | 8 | GAP-SCAN-001 | BLOCKED (2025-11-26) | Richgraph-v1 schema not final; Scanner workspace currently dirty, unsafe to land symbolizer changes. | Scanner Worker Guild | Implement binary/language symbolizers that emit `richgraph-v1` payloads with canonical SymbolIDs and `code_id` anchors, persist graphs to CAS via `StellaOps.Scanner.Reachability`, and refresh analyzer docs/fixtures. | -| 9 | GAP-ZAS-002 | BLOCKED (2025-11-26) | Dirty Zastava tree; need clean state to add runtime NDJSON emitter without clobbering user changes. | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint 0401 lands ingestion. | +| 9 | GAP-ZAS-002 | DONE (2025-11-26) | Runtime NDJSON emitter merged; config enables callgraph-linked facts | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint 0401 lands ingestion. | | 10 | SIGNALS-UNKNOWN-201-008 | DONE (2025-11-26) | Needs schema alignment with reachability store | Signals Guild | Implement Unknowns Registry ingestion and storage for unresolved symbols/edges or purl gaps; expose `/unknowns/*` APIs, feed `unknowns_pressure` into scoring, and surface metrics/hooks for Policy/UI. | | 11 | GRAPH-PURL-201-009 | BLOCKED (2025-11-26) | Depends on GAP-SCAN-001 and final richgraph-v1; pending stable symbolizer outputs. | Scanner Worker Guild · Signals Guild | Define and implement purl + symbol-digest edge annotations in `richgraph-v1`, update CAS metadata and SBOM join logic, and round-trip through Signals/Policy/CLI explainers. | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-26 | Implemented runtime facts emitter in `StellaOps.Zastava.Observer` (callgraph-aware NDJSON publish + subject derivation); added reachability options and unit tests; set 201-001 and GAP-ZAS-002 to DONE. | Zastava Observer Guild | | 2025-11-26 | Drafted runtime sampler runbook updates (config knobs, sampler rules, CAS trace pointers) in `docs/runbooks/reachability-runtime.md`; set ZASTAVA-REACH-201-001 to DOING while code waits on clean Zastava workspace. | Zastava Observer Guild | | 2025-11-18 | Normalised sprint to standard template; renamed from SPRINT_400_runtime_facts_static_callgraph_union.md. | Docs | | 2025-11-23 | Published runtime/static union schema v0.1 at `docs/reachability/runtime-static-union-schema.md`; moved 201-002..201-005 to TODO. | Project Mgmt | @@ -61,7 +62,7 @@ - Offline posture: ensure reachability pipelines avoid external downloads; rely on sealed/mock bundles. - Unknowns Registry shipped (201-008): unknowns pressure applied to scoring; monitor schema adjustments from policy team for purl/digest merge (201-009) to avoid churn. - purl + symbol-digest edge schema (201-009) depends on `richgraph-v1` finalization; may require updates to SBOM resolver and CLI explain flows. -- Runtime sampler code pending clean Zastava workspace; runbook updated so implementation can follow once tree is clean. +- Runtime sampler shipped in Observer; ensure `Reachability:CallgraphId` and Signals endpoint are configured per runbook before enabling in production. ## Next Checkpoints - 2025-11-19 · Runtime/static schema alignment session (Symbols, CAS layout). Owner: Signals Guild. diff --git a/docs/implplan/SPRINT_0513_0001_0001_provenance.md b/docs/implplan/SPRINT_0513_0001_0001_provenance.md index 7a93a7891..695025a2e 100644 --- a/docs/implplan/SPRINT_0513_0001_0001_provenance.md +++ b/docs/implplan/SPRINT_0513_0001_0001_provenance.md @@ -62,6 +62,7 @@ ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | +| 2025-11-26 | Attempted `dotnet test ...Attestation.Tests.csproj -c Release --filter FullyQualifiedName!~RotatingSignerTests`; build fanned out and was cancelled locally after long MSBuild churn. CI runner still needed; tasks PROV-OBS-54-001/54-002 remain BLOCKED. | Implementer | | 2025-11-25 | Retried build locally: `dotnet build src/Provenance/StellaOps.Provenance.Attestation/StellaOps.Provenance.Attestation.csproj -c Release` succeeded in 1.6s. Subsequent `dotnet build --no-restore` on Attestation.Tests still fans out across Concelier dependencies (static graph) and was cancelled; test run remains blocked. Need CI/filtered graph to validate PROV-OBS-53-002/54-001. | Implementer | | 2025-11-25 | Attempted `dotnet test src/Provenance/__Tests/StellaOps.Provenance.Attestation.Tests/StellaOps.Provenance.Attestation.Tests.csproj -c Release`; build fanned out across Concelier dependencies and was cancelled after 63.5s. PROV-OBS-54-001 kept BLOCKED pending CI rerun on faster runner. | Implementer | | 2025-11-22 | PROV-OBS-54-002 delivered: global tool `stella-forensic-verify` updated with signed-at/not-after/skew options, deterministic JSON output, README packaging steps, and tests. | Implementer | diff --git a/docs/implplan/SPRINT_304_docs_tasks_md_iv.md b/docs/implplan/SPRINT_304_docs_tasks_md_iv.md index 84b9e1cf8..7cf6e8a20 100644 --- a/docs/implplan/SPRINT_304_docs_tasks_md_iv.md +++ b/docs/implplan/SPRINT_304_docs_tasks_md_iv.md @@ -17,12 +17,19 @@ DOCS-FORENSICS-53-002 | TODO | Release `/docs/forensics/provenance-attestation.m DOCS-FORENSICS-53-003 | TODO | Publish `/docs/forensics/timeline.md` with schema, event kinds, filters, query examples, and imposed rule banner. Dependencies: DOCS-FORENSICS-53-002. | Docs Guild, Timeline Indexer Guild (docs) DOCS-GRAPH-24-001 | TODO | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Docs Guild, UI Guild (docs) DOCS-GRAPH-24-002 | TODO | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Docs Guild, UI Guild (docs) -DOCS-GRAPH-24-003 | TODO | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Docs Guild, SBOM Service Guild (docs) +DOCS-GRAPH-24-003 | DONE (2025-11-26) | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Docs Guild, SBOM Service Guild (docs) DOCS-GRAPH-24-004 | TODO | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Docs Guild, BE-Base Platform Guild (docs) -DOCS-GRAPH-24-005 | TODO | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Docs Guild, DevEx/CLI Guild (docs) +DOCS-GRAPH-24-005 | DONE (2025-11-26) | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Docs Guild, DevEx/CLI Guild (docs) DOCS-GRAPH-24-006 | TODO | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Docs Guild, Policy Guild (docs) DOCS-GRAPH-24-007 | TODO | Produce `/docs/migration/graph-parity.md` with rollout plan, parity checks, fallback guidance. Dependencies: DOCS-GRAPH-24-006. | Docs Guild, DevOps Guild (docs) DOCS-PROMO-70-001 | TODO | Publish `/docs/release/promotion-attestations.md` describing the promotion workflow (CLI commands, Signer/Attestor integration, offline verification) and update `/docs/forensics/provenance-attestation.md` with the new predicate. Dependencies: PROV-OBS-53-003, CLI-PROMO-70-002. | Docs Guild, Provenance Guild (docs) DOCS-DETER-70-002 | TODO | Document the scanner determinism score process (`determinism.json` schema, CI harness, replay instructions) under `/docs/modules/scanner/determinism-score.md` and add a release-notes template entry. Dependencies: SCAN-DETER-186-010, DEVOPS-SCAN-90-004. | Docs Guild, Scanner Guild (docs) DOCS-SYMS-70-003 | TODO | Author symbol-server architecture/spec docs (`docs/specs/symbols/SYMBOL_MANIFEST_v1.md`, API reference, bundle guide) and update reachability guides with symbol lookup workflow and tenant controls. Dependencies: SYMS-SERVER-401-011, SYMS-INGEST-401-013. | Docs Guild, Symbols Guild (docs) DOCS-ENTROPY-70-004 | TODO | Publish entropy analysis documentation (scoring heuristics, JSON schemas, policy hooks, UI guidance) under `docs/modules/scanner/entropy.md` and update trust-lattice references. Dependencies: SCAN-ENTROPY-186-011/012, POLICY-RISK-90-001. | Docs Guild, Scanner Guild (docs) + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-26 | DOCS-GRAPH-24-003 completed: created `docs/modules/graph/architecture-index.md` covering data model, ingestion pipeline, overlays/caches, events, and API/metrics pointers; unblocks downstream graph doc tasks. | Docs Guild | +| 2025-11-26 | DOCS-GRAPH-24-004 completed: published `docs/api/graph.md` (search/query/paths/diff/export, headers, budgets, errors) and placeholder `docs/api/vuln.md`; next tasks can link to these APIs. | Docs Guild | +| 2025-11-26 | DOCS-GRAPH-24-005 completed: refreshed CLI guide (`docs/modules/cli/guides/graph-and-vuln.md`) with commands, budgets, paging, export, exit codes; unblocks 24-006. | Docs Guild | diff --git a/docs/implplan/SPRINT_508_ops_offline_kit.md b/docs/implplan/SPRINT_508_ops_offline_kit.md index 8d6ac3956..70693b9fa 100644 --- a/docs/implplan/SPRINT_508_ops_offline_kit.md +++ b/docs/implplan/SPRINT_508_ops_offline_kit.md @@ -7,10 +7,19 @@ Depends on: Sprint 100.A - Attestor, Sprint 110.A - AdvisoryAI, Sprint 120.A - A Summary: Ops & Offline focus on Ops Offline Kit). Task ID | State | Task description | Owners (Source) --- | --- | --- | --- -CLI-PACKS-43-002 | TODO | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit) +CLI-PACKS-43-002 | DONE (2025-11-26) | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit) DEVOPS-OFFLINE-17-004 | DONE (2025-11-23) | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild, DevOps Guild (ops/offline-kit) -DEVOPS-OFFLINE-34-006 | TODO | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit) -DEVOPS-OFFLINE-37-001 | TODO | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit) -DEVOPS-OFFLINE-37-002 | TODO | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit) -OFFLINE-CONTAINERS-46-001 | TODO | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline Kit Guild, Deployment Guild (ops/offline-kit) -OPS-SECRETS-02 | TODO | Add Surface.Secrets bundles (encrypted creds, manifests) to Offline Kit packaging plus verification script. Dependencies: OPS-SECRETS-02. | Offline Kit Guild, DevOps Guild (ops/offline-kit) +DEVOPS-OFFLINE-34-006 | DONE (2025-11-26) | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit) +DEVOPS-OFFLINE-37-001 | DONE (2025-11-26) | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit) +DEVOPS-OFFLINE-37-002 | DONE (2025-11-26) | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit) +OFFLINE-CONTAINERS-46-001 | DONE (2025-11-26) | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline Kit Guild, Deployment Guild (ops/offline-kit) +OPS-SECRETS-02 | DONE (2025-11-26) | Add Surface.Secrets bundles (encrypted creds, manifests) to Offline Kit packaging plus verification script. Dependencies: OPS-SECRETS-02. | Offline Kit Guild, DevOps Guild (ops/offline-kit) + +## Execution Log +| Date (UTC) | Update | Owner | +| --- | --- | --- | +| 2025-11-26 | Wired Offline Kit packaging to include CLI binaries (release/cli), Task Runner bootstrap config, and task-pack docs; updated `test_build_offline_kit.py` to cover new artefacts. Marked CLI-PACKS-43-002 DONE. | Implementer | +| 2025-11-26 | Added container bundle pickup (release/containers/images) and mirrored registry doc copy; updated offline kit test coverage; marked OFFLINE-CONTAINERS-46-001 DONE. | Implementer | +| 2025-11-26 | Added orchestrator (service, worker SDK, postgres, dashboards), Export Center bundles, Notifier offline packs, and Surface.Secrets bundles to packaging; expanded offline kit unit test accordingly. Marked DEVOPS-OFFLINE-34-006/37-001/37-002 and OPS-SECRETS-02 DONE. | Implementer | +| 2025-11-26 | Updated Offline Kit doc (`docs/24_OFFLINE_KIT.md`) to describe newly bundled assets (CLI/task packs, orchestrator/export/notifier kits, container bundles, Surface.Secrets) and documented release-dir auto-pickup rules. | Implementer | +| 2025-11-23 | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild | diff --git a/docs/implplan/tasks-all.md b/docs/implplan/tasks-all.md index 17018a9ff..1894f64b2 100644 --- a/docs/implplan/tasks-all.md +++ b/docs/implplan/tasks-all.md @@ -696,8 +696,8 @@ | DOCS-GRAPH-24-001 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Graph Guild | docs/modules/graph | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Wait for GRAP0101 contract freeze | DOGR0101 | | DOCS-GRAPH-24-002 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · UI Guild | docs/modules/graph | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Needs SBOM/VEX dataflow confirmation (PLLG0104) | DOGR0101 | | DOCS-GRAPH-24-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Unblocked: SBOM join spec delivered with CARTO-GRAPH-21-002 (2025-11-17). | DOGR0101 | -| DOCS-GRAPH-24-004 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/modules/graph | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 | -| DOCS-GRAPH-24-005 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Wait for CLI samples from CLCI0109 | DOGR0101 | +| DOCS-GRAPH-24-004 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/api/graph.md; docs/api/vuln.md | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 | +| DOCS-GRAPH-24-005 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | — | DOGR0101 | | DOCS-GRAPH-24-006 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Policy Guild | docs/modules/graph | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Needs policy outputs from PLVL0102 | DOGR0101 | | DOCS-GRAPH-24-007 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevOps Guild | docs/modules/graph | Produce `/docs/migration/graph-parity.md` with rollout plan, parity checks, fallback guidance. Dependencies: DOCS-GRAPH-24-006. | Depends on DVDO0108 deployment notes | DOGR0101 | | DOCS-INSTALL-44-001 | BLOCKED | 2025-11-25 | SPRINT_305_docs_tasks_md_v | Docs Guild · Deployment Guild | docs/install | Publish `/docs/install/overview.md` and `/docs/install/compose-quickstart.md` with imposed rule line and copy-ready commands. | Blocked: waiting on DVPL0101 compose schema + service list/version pins | DOIS0101 | @@ -1099,14 +1099,14 @@ | GRAPH-API-28-011 | DONE (2025-11-26) | 2025-11-26 | SPRINT_0207_0001_0001_graph | Graph API Guild | src/Graph/StellaOps.Graph.Api | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. Dependencies: GRAPH-API-28-010. | GRAPH-API-28-009 | GRAPI0101 | | GRAPH-CAS-401-001 | TODO | | SPRINT_0401_0001_0001_reachability_evidence_chain | Scanner Worker Guild | `src/Scanner/StellaOps.Scanner.Worker` | Finalize richgraph schema (`richgraph-v1`), emit canonical SymbolIDs, compute graph hash (BLAKE3), and store CAS manifests under `cas://reachability/graphs/{sha256}`. Update Scanner Worker adapters + fixtures. | Depends on #1 | CASC0101 | | GRAPH-DOCS-0001 | DONE (2025-11-05) | 2025-11-05 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Validate that graph module README/diagrams reflect the latest overlay + snapshot updates. | GRAPI0101 evidence | GRDG0101 | -| GRAPH-DOCS-0002 | TODO | 2025-11-05 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Pending DOCS-GRAPH-24-003 to add API/query doc cross-links | GRAPI0101 outputs | GRDG0101 | +| GRAPH-DOCS-0002 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Pending DOCS-GRAPH-24-003 to add API/query doc cross-links | GRAPI0101 outputs | GRDG0101 | | GRAPH-ENG-0001 | TODO | | SPRINT_321_docs_modules_graph | Module Team | docs/modules/graph | Keep module milestones in sync with `/docs/implplan/SPRINT_141_graph.md` and related files. | GRSC0101 | GRDG0101 | | GRAPH-INDEX-28-007 | DOING | | SPRINT_0140_0001_0001_runtime_signals | — | | Running on scanner surface mock bundle v1; will validate again once real caches drop. | — | ORGR0101 | | GRAPH-INDEX-28-008 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Incremental update/backfill pipeline depends on 28-007 artifacts; retry/backoff plumbing sketched but blocked. | — | ORGR0101 | | GRAPH-INDEX-28-009 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Test/fixture/chaos coverage waits on earlier jobs to exist so determinism checks have data. | — | ORGR0101 | | GRAPH-INDEX-28-010 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Packaging/offline bundles paused until upstream graph jobs are available to embed. | — | ORGR0101 | | GRAPH-INDEX-28-011 | TODO | 2025-11-04 | SPRINT_0207_0001_0001_graph | Graph Index Guild | src/Graph/StellaOps.Graph.Indexer | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. Dependencies: GRAPH-INDEX-28-002..006. | GRSC0101 outputs | GRIX0101 | -| GRAPH-OPS-0001 | TODO | | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | +| GRAPH-OPS-0001 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | | HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild (ops/deployment) | ops/deployment | | | GRIX0101 | | HELM-45-002 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Security Guild (ops/deployment) | ops/deployment | Add TLS/Ingress, NetworkPolicy, PodSecurityContexts, Secrets integration (external secrets), and document security posture. Dependencies: HELM-45-001. | | GRIX0101 | | HELM-45-003 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Observability Guild (ops/deployment) | ops/deployment | Implement HPA, PDB, readiness gates, Prometheus scraping annotations, OTel configuration hooks, and upgrade hooks. Dependencies: HELM-45-002. | | GRIX0101 | @@ -2905,7 +2905,7 @@ | DOCS-FORENSICS-53-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Timeline Indexer Guild | docs/modules/evidence-locker/forensics.md | Publish `/docs/forensics/timeline.md` with schema, event kinds, filters, query examples, and imposed rule banner. Dependencies: DOCS-FORENSICS-53-002. | Requires timeline indexer export from 055_AGIM0101 | DOEL0101 | | DOCS-GRAPH-24-001 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Graph Guild | docs/modules/graph | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Wait for GRAP0101 contract freeze | DOGR0101 | | DOCS-GRAPH-24-002 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · UI Guild | docs/modules/graph | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Needs SBOM/VEX dataflow confirmation (PLLG0104) | DOGR0101 | -| DOCS-GRAPH-24-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Blocked on SBOM join spec from CARTO-GRAPH-21-002 | DOGR0101 | +| DOCS-GRAPH-24-003 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Unblocked: SBOM join spec delivered with CARTO-GRAPH-21-002 (2025-11-17). | DOGR0101 | | DOCS-GRAPH-24-004 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/modules/graph | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 | | DOCS-GRAPH-24-005 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Wait for CLI samples from CLCI0109 | DOGR0101 | | DOCS-GRAPH-24-006 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Policy Guild | docs/modules/graph | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Needs policy outputs from PLVL0102 | DOGR0101 | @@ -3319,7 +3319,7 @@ | GRAPH-INDEX-28-009 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Test/fixture/chaos coverage waits on earlier jobs to exist so determinism checks have data. | — | ORGR0101 | | GRAPH-INDEX-28-010 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Packaging/offline bundles paused until upstream graph jobs are available to embed. | — | ORGR0101 | | GRAPH-INDEX-28-011 | TODO | 2025-11-04 | SPRINT_0207_0001_0001_graph | Graph Index Guild | src/Graph/StellaOps.Graph.Indexer | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. Dependencies: GRAPH-INDEX-28-002..006. | GRSC0101 outputs | GRIX0101 | -| GRAPH-OPS-0001 | TODO | | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | +| GRAPH-OPS-0001 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 | | HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild (ops/deployment) | ops/deployment | | | GRIX0101 | | HELM-45-002 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Security Guild (ops/deployment) | ops/deployment | Add TLS/Ingress, NetworkPolicy, PodSecurityContexts, Secrets integration (external secrets), and document security posture. Dependencies: HELM-45-001. | | GRIX0101 | | HELM-45-003 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Observability Guild (ops/deployment) | ops/deployment | Implement HPA, PDB, readiness gates, Prometheus scraping annotations, OTel configuration hooks, and upgrade hooks. Dependencies: HELM-45-002. | | GRIX0101 | diff --git a/docs/migration/exception-governance.md b/docs/migration/exception-governance.md new file mode 100644 index 000000000..74e376888 --- /dev/null +++ b/docs/migration/exception-governance.md @@ -0,0 +1,64 @@ +# Exception Governance Migration Guide + +Status: Draft (2025-11-26) — aligns with Excititor VEX/exception flows and Doc task DOCS-EXC-25-007. + +## Why this migration +- Retire legacy suppressions/waivers and move to policy/VEX-first exceptions with provenance. +- Provide auditable, reversible exceptions with rollout/rollback plans and notifications. + +## Target state +- Exceptions are recorded as: + - VEX statements (OpenVEX) referencing findings and components. + - Policy overrides (simulator) with scope, expiry, and rationale. + - Optional authority approval attestation (DSSE) per exception bundle. +- Persistence: Excititor stores exception records with tenant, source, scope, expiry, approver, evidence links. +- Propagation: + - Graph overlays surface exception status on nodes/edges. + - Notify emits `exception.created/expired` events. + - CLI/Console consume exception status via `/exceptions` API. + +## Migration steps +1) **Freeze legacy inputs** + - Disable new legacy suppressions in UI/CLI. + - Mark existing suppressions read-only; export CSV/NDJSON snapshot. +2) **Export legacy suppressions** + - Run `exc suppressions export --format ndjson --with-rationale --with-expiry`. + - Store exports with SHA256 and DSSE envelope in `exceptions/export-YYYYMMDD/`. +3) **Transform to VEX/policy overrides** + - Convert each suppression to OpenVEX statement: + - `status`: `not_affected` or `under_investigation`. + - `justification`: map legacy reason → VEX justification code. + - `impact`: optional; include nearest safe version if known. + - Generate policy override record with: + - `scope` (component, environment, service). + - `expiresAt` (carry forward or set 30/90d). + - `rationale` from legacy note. +4) **Import** + - Use Excititor `/exceptions/import` (or CLI `exc exceptions import`) to load transformed NDJSON. + - Verify import report: counts by status, rejected items, conflicts. +5) **Notify & rollout** + - Notify downstream systems: Graph overlays refresh, Scanner/Vuln Explorer respect VEX, Policy Engine caches reload. + - Announce change freeze window and rollback plan. + +## Rollback plan +- Keep legacy suppressions export. +- If import fails or downstream errors: + - Disable new exception endpoints feature flag. + - Re-enable legacy suppressions (read-only → writable) temporarily. + - Investigate rejected items; re-run transform/import after fix. + +## Notifications +- When migrating: send summary to tenants with counts per status and upcoming expiries. +- Ongoing: Notify events `exception.created`, `exception.expired`, `exception.rejected` with tenant + scope. + +## Validation checklist +- [ ] Legacy suppressions export stored with SHA256 + DSSE. +- [ ] Transform script maps all legacy reasons → VEX justification codes. +- [ ] Import dry-run produces zero rejects or documented rejects with reasons. +- [ ] Overlay/Console shows exceptions on affected components. +- [ ] Rollback tested on staging. + +## References +- Excititor module: `docs/modules/excititor/architecture.md`, `vex_observations.md`. +- Policy simulator: `docs/modules/policy/architecture.md` (POLICY-ENGINE-30-001..003). +- Graph overlays: `docs/modules/graph/architecture-index.md`. diff --git a/docs/modules/cli/guides/graph-and-vuln.md b/docs/modules/cli/guides/graph-and-vuln.md new file mode 100644 index 000000000..aad378531 --- /dev/null +++ b/docs/modules/cli/guides/graph-and-vuln.md @@ -0,0 +1,82 @@ +# CLI Guide · Graph & Vulnerability + +Status: Draft (2025-11-26) — reflects current Graph API surface; Vuln API pending (see docs/api/vuln.md). + +## Prereqs +- `stellaops-cli` built from current repo. +- Auth token with scopes: `graph:read`, `graph:query`, `graph:export` (and `vuln:read` when Vuln API lands). +- Tenant header is required for all calls (`--tenant`). + +## Commands + +### Search graph +```bash +stella graph search \ + --tenant acme \ + --kinds component \ + --query "pkg:npm/" \ + --limit 20 \ + --format ndjson +``` +Outputs NDJSON tiles (`node`, optional `cursor`). + +### Query graph with budgets +```bash +stella graph query \ + --tenant acme \ + --kinds component \ + --query widget \ + --include-edges \ + --budget-tiles 200 \ + --budget-nodes 150 \ + --budget-edges 100 \ + --format ndjson +``` +Returns nodes/edges/stats/cursor with cost metadata. If edge budget is exceeded, an `error` tile with `GRAPH_BUDGET_EXCEEDED` is returned. + +### Paths +```bash +stella graph paths \ + --tenant acme \ + --source gn:acme:component:one \ + --target gn:acme:component:two \ + --max-depth 4 \ + --include-edges \ + --format ndjson +``` + +### Diff snapshots +```bash +stella graph diff \ + --tenant acme \ + --snapshot-a snapA \ + --snapshot-b snapB \ + --include-edges \ + --format ndjson +``` + +### Export graph +```bash +stella graph export \ + --tenant acme \ + --format ndjson \ + --include-edges \ + --kinds component \ + --query "pkg:npm/" \ + --out graph-export.ndjson +``` +Manifest is returned with jobId, sha256, size, and download URL; CLI downloads to `--out`. + +### Vuln lookup (placeholder) +Pending Vuln Explorer API; for now use graph search/query to inspect impacted components. See `docs/api/vuln.md`. + +## Exit codes +- `0` success +- `2` validation error (missing kinds/tenant/scopes) +- `3` budget exceeded (error tile received) +- `4` network/auth failure + +## Tips +- Add `--format table` (when available) for quick summaries; default is NDJSON. +- Use `--cursor` from a previous call to page results. +- To sample overlays, pass `--include-overlays` (budget permitting). diff --git a/docs/modules/graph/README.md b/docs/modules/graph/README.md index f5b1a4778..d84fd3711 100644 --- a/docs/modules/graph/README.md +++ b/docs/modules/graph/README.md @@ -35,11 +35,26 @@ Graph Indexer + Graph API build the tenant-scoped knowledge graph that powers bl - Logs/traces: structured ETL logs, query planner traces, WebGL interaction telemetry (once UI lands). - Offline bundles: deterministic `nodes.jsonl`, `edges.jsonl`, overlay manifests + DSSE signatures, consumable by Export Center and CLI mirroring. +## Operations & runbook (Sprint 030) +- Dashboards: import `Observability/graph-api-grafana.json` (panels for latency, budget denials, overlay cache ratio, export latency). Apply tenant filter in every panel. +- Health checks: `/healthz` should be 200; search/query/paths/diff/export endpoints require `X-Stella-Tenant`, `Authorization`, and scopes (`graph:read/query/export`). +- Key metrics (new): + - `graph_tile_latency_seconds` histogram (label `route`); alert when p95 > 1.5s for 5m. + - `graph_query_budget_denied_total` counter (label `reason`); investigate spikes (>50 in 5m). + - `graph_overlay_cache_hits_total` / `graph_overlay_cache_misses_total`; watch miss ratio > 0.4 for 10m. + - `graph_export_latency_seconds` histogram (label `format`); alert when p95 > 2s for ndjson/graphml. +- Triage playbook: + - Budget denials: lower default edges/nodes budget or guide callers to request smaller scopes; verify overlay includes are truly required. + - Overlay cache misses: ensure cache TTL is ≥5m; check overlay service connectivity to Policy Engine; warm cache by replaying recent hot nodes. + - Export slowness: reduce export `Limit`, offload PNG/SVG to worker, and confirm disk I/O headroom. + - If alerts fire, capture tenant, route, cursor/budget values, and recent deploy SHA in incident note. + ## Key docs & updates - [`architecture.md`](architecture.md) — inputs, pipelines, APIs, storage choices, observability, offline handling. - [`implementation_plan.md`](implementation_plan.md) — phased delivery roadmap, work breakdown, risks, test strategy. - [`schema.md`](schema.md) — canonical node/edge schema and attribute dictionary (keep in sync with indexer code). - Updates: `docs/updates/2025-10-26-scheduler-graph-jobs.md`, `docs/updates/2025-10-26-authority-graph-scopes.md`, `docs/updates/2025-10-30-devops-governance.md` for the latest decisions/dependencies. +- Index: see `architecture-index.md` for data model, ingestion pipeline, overlays/caches, events, and API/observability pointers. ## Epic alignment - **Epic 5 – SBOM Graph Explorer:** Graph Indexer, Graph API, saved queries, overlays, Console/CLI experiences, Offline Kit parity. diff --git a/docs/modules/graph/architecture-index.md b/docs/modules/graph/architecture-index.md new file mode 100644 index 000000000..8d6d88863 --- /dev/null +++ b/docs/modules/graph/architecture-index.md @@ -0,0 +1,48 @@ +# Graph Module · Architecture Index + +Status: Draft (2025-11-26) — aligns with Sprint 0141 (Graph Indexer) and Sprint 0207 (Graph API). + +## Data model (canonical) +- Nodes/edges follow `docs/modules/graph/schema.md`. +- Tenancy: every node/edge/snapshot/event carries `tenant`. +- Snapshots: immutable bundles of nodes + edges; overlays stored separately (`graph_overlays_cache`) to avoid mutating historical snapshots. + +## Ingestion pipeline +1) **SBOM snapshots** from SBOM Service → Graph Indexer (`graph_snapshots`, `graph_nodes`, `graph_edges`). +2) **Advisory/VEX** from Concelier/Excititor → edges and overlay seeds. +3) **Policy overlays** from Policy Engine (POLICY-ENGINE-30-001..003) → cached per node in `graph_overlays_cache`. +4) **Runtime/signals** (future) → additional edges/attributes; stored alongside snapshot edges with tenant guard. + +## Overlays & caches +- Overlay types: policy (`policy.overlay.v1`), vex (`openvex.v1`), analytics (clusters/centrality). +- Cache TTL: 5–10 minutes recommended; deterministic IDs (SHA256 over tenant|node|kind). +- Analytics overlays emitted as NDJSON (`overlays/clusters.ndjson`, `overlays/centrality.ndjson`) with stable ordering. + +## Events & change streams +- Inputs: `sbom.snapshot.created`, `advisory.observation.updated@1`, `sbom.version.created`, `sbom.asset.updated`. +- Outputs: Graph Indexer change-stream/backfill emits node/edge upserts and overlay refresh notifications for Graph API caches. +- Downstream consumers: Scheduler (recompute jobs), Graph API (query caches), Export Center (offline bundles). + +## APIs (reader surfaces) +- Graph API (Sprint 0207): `/graph/search`, `/graph/query`, `/graph/paths`, `/graph/diff`, `/graph/export`. +- Saved queries and snapshots exposed through Graph API; RBAC scopes enforced (`graph:read/query/export`). +- Health check: `/healthz` (200 when service ready). +See also `docs/api/graph.md` (pending) for full endpoint parameters and error envelopes. + +## Observability (current metrics) +- `graph_tile_latency_seconds{route}` +- `graph_query_budget_denied_total{reason}` +- `graph_overlay_cache_hits_total`, `graph_overlay_cache_misses_total` +- `graph_export_latency_seconds{format}` +- Ingest/backfill metrics live in Indexer (Sprint 0141). + +## Offline / bundles +- Exports: deterministic NDJSON (`nodes.jsonl`, `edges.jsonl`, overlays) with SHA256 manifest and optional DSSE. +- Air-gap friendly: no external calls; rely on cached schemas and local snapshots. + +## References +- `architecture.md` — deep dive. +- `implementation_plan.md` — roadmap and risks. +- `schema.md` — canonical node/edge shapes. +- `docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md` — Indexer sprint. +- `docs/implplan/SPRINT_0207_0001_0001_graph.md` — Graph API sprint. diff --git a/ops/devops/release/__pycache__/build_release.cpython-312.pyc b/ops/devops/release/__pycache__/build_release.cpython-312.pyc index 798db2d642d08e21314ae644a78c5c46321e3d84..1dab254092e6df3b0457b1485ebfe0db9b8696aa 100644 GIT binary patch delta 22 ccmdnLk9q$-X71Cxyj%=Gu$E6@Be%(Y08~{5r2qf` delta 22 ccmdnLk9q$-X71Cxyj%=GaOD{5MsAb)09h{wX#fBK diff --git a/ops/devops/release/__pycache__/verify_release.cpython-312.pyc b/ops/devops/release/__pycache__/verify_release.cpython-312.pyc index 832dfb120b8cd2b9511a39da7a8bbdca73aef675..7a0f262d6c5e2cb36c8683ce286e9a8ee34f64ad 100644 GIT binary patch delta 22 ccmbQ-z&N>qk^3|+FBbz4tmRYK$j#vZ078}p0ssI2 delta 22 ccmbQ-z&N>qk^3|+FBbz4Tsg+Nk(Hj1$9 zG_#F&y&F4sXG5m54e4YOx=tr#)n<}EsoULj(n-_dC3s9VZtb+4rZa6*JG9f8%w~4a zy-z~2+dR-Y=bf*6?!D)nd*2U!jQ)BD**?h0v2ySTOVxq=t{(#%B!UkAlTc;B-DbG<_F)#U_w-iG5--;BGP2FSY>D^Y@gQ z!GPEb@?PuG5u<=Q`5x(`a7r zJ{%c3BKW2SzvNOTCD(`|1%pETUA;?J-q|=4QW~X3QJQP?2b2bx$voj%xgm`Gjqy8% z54WK2kng%`(Q9Pg^@QPD;P?enQCSr4sW^k6p<7yRU3l8LH!tJe6b+kF1N?OB{t z9ov`9s$=JBORwtaec!w~GNJZQ+~oaaKHpi;K!f3;oQUY)3%aOo3XyyH&KA3(*8+f$ zbPyDir-K5Hjs>k}cKWD(%0TjR3S<21hO}1Nhhj{56OGKyL}64fGMu&mzh>r<|GTzK zCR&&+ui>;UwTNlMWDKX7SlOUiNyCh{wE29tJ0~*K$=1}>H8kKl69@@7Aj$PEMUbam zvycETG32^A))sL0}0ESa9EKQ z3_F)aJ$W#*8pQ$xo6t*&FH)SrZ)K%gyNO4FHu0fX@=g-$nYDWBO3s?=!2`NNtrJ^JMX1- zFkx?onIOxOBI9>J>0N4f9l%BIuI+1!^*ySs;$q*r$@Yy4moBWCif@~XuAW=UT9#B- z%bL0MuCsdCsXAL%7F1_9;cHy6)g1>_$HCRkvDIT(?Zj)meAm8ZaZI&WUF^?D;cV5> zx-xmARNZxWjUUbsSX4*T$`H7?QS<&WmAaACPU$8;fwx0czGN8z$w3d{ILHySaRZ2C z){}p(u}r1%Tg&NGHHa*?H)YyevQk8zrBbpkl5Z)Q;Q<+Y3-HO>I#|3lmd9}?Ag%@d zLPnNH^-(^Zj8_~yH*a*CB4b^!2d*&oO-eL{lrUtQgy#Z2sor%ijHe;{M!E;oQn}=5)A6J{l*Ub)15>?GDE5~o_SKA(69X+kKo!*p0RCCix z`;9WS<+0T#PN*#>HYHKD{UM`!7K7b*vF~0sSG4tolh;lX@8=vcR9?0`tABQs=C8ci zebZcVUk^;$G)spd!WTFNTBr7;5{9M<8LBLa0fTi%U@)BCS2#=f$N<~Zp7XO|thl;h zH((QKOBoI;fr$XTgeN9~fso`%Hh3mf!eQX~)7waQF_I%OFCHKBk}!v;ix~JA{tjY0 zgkOaiWm*NPsxD_$t=X}{W;S4w+KC^7^3SPNEr5&MZ}|MH?Mvlwguo`))#k_6_#;q% zz}uoa>Xwhcy;t@0+~j-H^+vv)pBIBe^(z;N)wcd{V=R zAucX4TQWGPv3~qPC9QEf+MDtB>03+nlmW*nwOsw* zhE*Vb6QAAj5kjw%F87y)Tfq2hyr=dvoh}EFHqVOj7r-%~`0qXIdgE__E=2L4G<~Ww zZU9|{$Xl&%ktYiZYfn9JI;XPAuY+Ea>^uwlsIHqk`6Bt3wk>2O-<&U_D+TGJGs?4L z+2Pcut}`hNMgeF+hNyuEwo)@SiyEfPaE2G`oa5lN3L27O3O4HD2ijCJ^XWD&7^6mV z#8IG|vV*OI+}WB--fY|&iyETF^cG*4=D921gqCJIbj~t~HxG zo!(nlAjIxlSPDvlEV;x0mVD3x=Nl5%O?x?E49vjIOpfriwT-p=>AmHPn7+Xu`GRjy z1WchnqMf$GZmV`uJ_kw(-X9E~!RLX*zoy_16wFh=j!=nG5ddye#yM*BL8%5rL6Kxk zTUCO`aJ(msTCFDFgvWkKiFs!QWl}bQ0zFUR(T>z2mV%qHdy^H8{|^**!?Mc%2?QLa z`CFH=)%^P9>9_mUhVIq8o{NXp^GX+^YF-^sxm!NxOorxX`E|SFxu-5ab!A~ayWnEh^1$6h?PH23k=P{YSXjh}P8rTN+3b$j7+7cO78X)j;ueeLkehgaK%KH4_4URJZb z>sDQdTG#Qm_;FpATGn+nYdzO_E4Nb3t-O_6qvqDE6x`rfb8FUedv50zy^wt^dvR>( z_|4p=`xbC~-vl9OMbDF$n+xKX+79b1_aT1#B><9mzX|qmEiA_!R|gi}Miq^58!hc7ggOB0u_` zuF@Re+u5ZHqo^*Ou8Ic?*b{w_-}n{$rmx`B(b@BmpEh#MSco2>(7tC=M;GO$_L8f; zg=io7pKdqWOFrxVTHC&qJdMA}{Rh8bQ4XXya6{KeE!S!0!?R#j9wqyFDo_`B^FXO? zs)s~-oT!&9_SD%2G!?jdTVb$6Y_K5)hZ#IVUhOS6*`hXVQ%1<%UT21d=_@U^qY?7c zUT59Wly}hvx?`!gG(L~yY6PDfFin^mO!KquqB7-JO5u8vfcY6FbfP_Ko$4j0`^wSr zYpYIl0?0x`e$>R6h2(?2+zf%fFv^ireZ__%<{nh_#mX_#-CvdOP5F=7lC6v6^%jzU z?a#>&*)v5y>USc6_Oq={(ek=Al^Jc^LcZcld7Bc+N`C>V9B?2%>wNb2yuVbT;$2i87igXrZ8$JbCDuY#k+8E$I>Y z^w5E9BW2J&jjf0@5AJlevTl_y!BsCDoRiq~AJ@Lr`5T+L~Oz!~fWW@BS!t{*7v}m=FSq!uU@gNnj^2(#Hr|9c0k~QRu{fUzID4<^z zxQ7B8)v;5M3&8D2@X##fbaU0BETjtg6wr={9RLyq$t#gJbp`OslLB^Ac`cP&wMI(0 z2?wRgABPsKV<37Fe)8{#a`=;-l}qg_S!?{h+x(Wr_GO@2Z}X*#$CvBY_+7X8s-?jd z=NjK}n=e^(E={lT&GC`pJG!i&mDL_G=vNJfBer+jxRTFizeY zeH9ImlSg-y;~&zAjsrWA=kfT$a=;(*%)%WP?$sh5B(EIpM2EEe5y?9dw-V~j+{n~Z)c8z%AbK3%yk{=ehL(BK?p(JlwK@HEY93s3nEaQi8PEkD&w z@WbJdlrlU=mQHLjG9Ty2jWl`pM6;lA!2~`mFFoJ`pVPQ&f6K)Q;JJ+%2J|e5xQECmr;3YCQ|9Azg^y6sMS%?wo442)rlM+6=Y0(Ik*B;B#=BJb3Hi3SHp@q4 zA_Wq;zeA~q>Y7zOy_sfI_y&EYR}rn&>Cuc?neU?17zGo=f4ar`4Its4 zli!HHa{5EmVL_8-Em!=%tx`~PRwcZvW2 delta 2316 zcmZ`)eNa?Y6o2>a+qbaG0`etaDu{rK^8Fzq2*UDZDgnD9IELU|Y?p<_%hKqCA307l z^-tOP(?6PK8vlqg>oqn_Njgqi8Dlz`x}%9RX-%5ipfzKdo%1$!^^YFB-#PbpKJLBe zp1bd_>+H+ZEc&d~YL@UBF~zy-S`S7ivppZ>*2{a#c^d|tuRQ5NtO zd;!WrUdb1tEaEHqB9z4_Gf?ujchUA7w3X z-~}k_(3rFVr?K-w9IoMwya;7IMmOW-{{FVOW^i!3MM)ncJDNOR_~lLe?JmAgjfE1S3Dj zcXSe-WK9|_Djt|FO4H6Pf14Qt2A6*u3>MT`-$##tX~@UNyslCBq`+edME3i9{!zc^ z8-c9CQ^wEG8MpGNYNY+)Jf{E6lbsvR5c5<9tDfPxx$Ha|KI}*`?_{GnOwxB7-W_TN#oq?I zJJVP-6gv{(NT->#LuPTR?CBuF7&mUhy^%?A#6E3G!Ripb)uCX;P&3S@OpYw(gm)bA zn2+lDwIeFgIk%MY&@wS4S!W%otV^$Go-@wUJ?H1?(XBns1hz%D?sukJ!y8)YZ-oix zf|T&~O>^5Hka-4LZ#ZAexsBPxZv^4)QA7-Og@`4XM-T^ZZafe*OIN-)?}lu;=5;V57du2G-&C6{=GY$|1lC+!4$-n9fT@^n{j4P9D6H%`!<5_Fja zU54NU)1*;n3))YxYwwZv0zm?7*|aU=GO1Sxj0BAYcL;tbxGTZ*ro<$D@);uT6C_c} zKP0epr>G{yX(zWl zlvyQwy0sGV{#LtMMM)~5s_!t%g&Tdj5dnp}@F%4bV*2OfFIR4VJzEN%elPl8>JL~N z$)Dgf)4teNV^DRX44w=Wt4(BE4i(#*<(o{~ySjj23Hq=Ywl$1|3BUNHoc$polKKm*XXDoS=&bZZr^a9cQclAi*Uj^Px9Vb zvc&lN#DLrDDsc3P*5LtPum+oGgAq^7q947!K>eh{)gsOoCBoIxyS)qJgDtOXdf)3> z92x=&ek#1es&C3cGbwmd_dT?W$ia*k>F6HChgR%$kNWjx{VCwSrLgCOWah(<6Bm6``o#eZoO zw7Q`wmKnhY*-TO_YhW`u7S_RL(yeUeOooN6evlr=;_v6dTHkK9mofE$9j1JF{{pwD BKeqq? diff --git a/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc b/ops/offline-kit/__pycache__/mirror_debug_store.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9097892d63f9795c1f440f93a0f8efbd8dd09bc GIT binary patch literal 10973 zcmb6A~?4SL< zBV9?xIB5^VIY;Mw=k=ZMe9!xzR;!tU^g|mL{GfrN{(wK!#N;dV{brh?rYW8ppm>_s zgz1Q8Ktp3)8`chJNy!Z8NU0ytlhQC?Af<7@NJ`Uy2}&kxj<5r4#4=!^DGenULbd8* zrBbkc0PA52tEd0y2}(Eb z=FL!6@l`wvWi?;TTcGst9^MLN4PV38^0wD>1GP}A<13(6C)D%yORW?oT4CP}@L#9y zMBVMH^r+Tb@4o??KN1whn8=AjSnx{%#|wk;i|vve69vvM%EFL8AWK{@D#ti^Sm2Hg z4TXbIfjbnGIZ5_k3`Q?xAdwO3#23iSofkMhC<+0X zf0SztMg!qEk8@n;7>ozQe0z}Zys*Wbhi#Aeqro9Tk~>0DEP7!J=a2Ga$_pLRuz%}Q z+b_U6kr4ojMZIfo(Hx7*BXPNGNu1)cAO?rj?Wt>U!+vQPw(K90VgGnkIKyZM*UL%q zNW?FWn)z5D9ucB42gkf324z`@a`7k+&=(?t?C1Tmzq1T``Au_Dj2rO>F6A*5iVaF! zz#lbBaS;$Jkahf*{lTz*Ff4E?3SbwwO@BBDlfvN;({fzuFn@})>` zUX5bxhfU)Yitbc=BrJTY0k}YcGI0#0hkq?a(f8poP_HxaLTH`}Q)6_J4nh0sq|(cn zCP{^Kt9vPm*Qk{A6!lF;HmDF2%n-b^*Ni;#0d4`(Uh2G)qQ)=#MjQ;3 zPmj2O{Q@6>%6I5#YLW_-c^jEt@rd&2(6 zAn)IuAVeRR1z-LVbo7K{0e@K94V63u@J33qL-yx!YNq)IZQpB~y_9Y29;fc(?nN79 zUcF*Kf)57=6@7jlg^7&_QH2QtDJlB+2#}tjn1+R`eDET?L9b3>M*Q-ySOLQoEsRlg zf#G=cl8DzMA&qd5o)azDZ^MUK+N0=!M*NqekioV-W*m3TmDf5ZJF@25@jW?{^;*qj z&D4``tY4-yx^=1jOYWL%ZTI}1toylp6m42Ju`g$-xOQ;z;8gsMrQtIzG<~iG$YnhR z?axd;^TxAjruK8gBTXw3?(YZ9z0KNRH0yeI8`gdnF9Far1;oDQW%HUFT9qW9f=444 zpG>GAcwl%jC5yxn(5xosM+tV6Dv%^|RH4NcU0fb&fBJn|F$&Q@40)qhD^|kX3WHow zVZt#WF^ytX8JCaT3b_QNXpYO@fED3@H6fA*p>ha#-Qy5l;!ePQtAYD|1(%oTH(g2kpXg~C zxNr?ajZ#0<0*}-HL2d>-=){ZR*q~wvO1`0BSU}7Sf~w#HK`y%Syyb|oBmXEcFNBuF zP5|jbuq}}N5nq4K?wPh-x22@4z3Cpj_v#5I2dD3u?D^L2oTKWEgP-Z3@e4w3uWst3 zXAPZn02jf84-_w*klXj60$9+9PPzDSc{)jb7bH}YhFtruPUX}zoa@V_o#>W}vMz)Y zb5$xmY?>;A#8NfZ!8jF!9`V z0GD&7lj$Bk5#&nN?VAN|L(?15HNEl;=q)yZ%_3#XKeFbMP zjWGf#twkcn1hC?@R2`KojhT|BFH>nGn`}kYVdC{k<9Dfdi#+jFT1N3pQqojBp*d+z z>My}%iY^&N1xdUC$MVLc9&%IC4D;yK5wFro!zU9fD8Y7f!qpKlR0N(&7`H$p==xm3aS~>J81z%*JYI{U zg(s*OP#U9FWj!7jh2t?Pcony$=s{k<{uP5LNU<=SkG$~8{lFoQ#^n7t`98G6h^C~N zU+{zE_{KxIS^#E;?H?&eqx5tn_=>mf6;fYwMzOSH`(3>wIRyc(rU0*W;ZQ4>T{0fnX5CQId=Z={O((x^oeun=g+6@-?*>UIBlS=T5Zcl z%2JbNH|A;^-)n!n{kxruwOyIou8BjrirQ491c% zpYo?&gD^|A17@joz$~W9soqKJq;Yu%pkcY2vber|GF9`|H_~iVnrX_jR|fE?CSWyr zIam*^(^N?YmY&vL>KYT!T%-aTVuv9^Eoab$nRuG`3p7dQx(4mAG2Iwbl5o7{H7cpw zTU4x6HZrC!S$?3gP z#XT8i;w8d^g^)s5rBIQ9mO9EtAp2OALXK4p@bgAd5QIO`SNwN@&VAcXm<8`U46Vg?E3HHY9OtWLZ~JstQhS9wE5 zf!1lHcjk&(GnCg>dqaxGWzErqzJo}CM@))sXi^-9U8p9-`E~F-(PTiuwkj1#cMT>| zFd#V9fOww<-;{{zrFZ}`up2ZTPds8VteAQ(3Q-V4;_gIa-q!}grQrA|`8HIT2+Sm* zAxA8U+pgZ(Q&XoeLr$A0@_A2c11B_H4%%#&$t{w6$UU70OwFpiXm$Jve*Yh#joLb zUE~tlg{X##Ku4g9@$m>4)#x!5ysX%dZ5H*=#BM%FnhI{#ygLIKqr&F37Ggyz=DZGw zI*@o6drbr>z93-5P#BwcMWIkE>LjZDQ?ZJ&#J>foA^4YW5tZbDo^o%RIhS>9{a+p9 z)Zew((IVY7xhvIrn{B+OqfFLiEp&_@TBe|QV4)mtptQNO3$EU5W$*aGC3{t_ZQF-E z@Au5JH&0~S`hIKl&uqH2?>GIw?oYqqyL0%$gq!%dcHS{H+&20_?pNx^`yZGoPwn)J z*I%4|`TEN@8b54%zwN`$_dBOv&Q^E-&SamgPCY+Soi(lhoyjp-^F|$Mn)0N!K~rvh z%X`VUle0&%>$`Kc%{h0&^ttQjX4?n)m&k==gsw~ol{ z*VCuZEuOxRIej7RzjWtxc#R9)7+DYvc_TB^;TH8?8t;|G>oDP83vQ=MU| z7nz0((=fx&ok}we3ru&;f!At+{lr|CUia)xX8!Wc z=hO8EvgW=t)Ay+J3Rw}s?>TsH<*Mphn3IuzlXRo#OC>i^RAkhC3`NLA2TZ7_`jz_S zq7uk!@kV)uA zNlH~2z>c;b1(pxIQwo4L6GKD^YkPa)(%a#>6)i-?kc*RmU*h12Lgep}SQ!XL*&mFO z5FfgVY!>zq*lF>7yMF;JUSvMhC;W(U_3BU+1_3j;J`g3*s_Mi zlgHMGDgu-mCR}qdJXPM*Y>!35VB?aV`D8IJBzjL?3XTx}8nO4ma7U1Q;BS-!-H5!z z&cZ%8h`f}a5MGlWY+b;fI0dwnFtxWAymyJF3!cI=d5CUYPZ zDi{oumx(h0CMtj4;@dEqWGSdfKZk6bTC%zqts67ejX$YguqjgD`(yF81U*`MP1H% zeAz%IV_-6K*zI;8@nF)JG!bXx zzOs|B#Mx*`n!wp;eOLFFp9p*jYgk)k9vkexS=~jcgcT~@WqA9I$`W2=R53_O!cC~{ z=KybG<}tQxpAK}n^3Ei(jtZ%iNTd{68Z7)Hyw$V!q{>ytl9=8f33`Yiti#}D17&k@LutYD3`jq zBJjz+-jZ9k6+}l&q6tJ~0vlR#=o*zP$M-j$ktqbiwURj5HDq75^Sv(QivEC9hTdXQ&hTBPo_b@QIvH z(F#`sA_N~Ok;AMX;VMYnOt&T0J;i6)y79fs45O=Ds(xb5wb<30>FQnV>d$obXS)t%s}E0DKnA%y zi_Vsevt<^tu0=<8#?hU1^o$>Z*y|!&cbkRSmM2v|^Wyy8e?R=Q!*?L`g<;8W_T-$d zyRP-=#vK{gj`1To*0ad2&#>zk*bO9RwqxEp_j1~H;J*UD3H>^B$9M+Z*`HL@r_N_9 z+Q6;t*|g~C%6Piw+84L(&ura)t0uGcMAmaM&A6AGo6^lYvd*W+4=;i5`AF8ib^Pc@ z24~7LYrE~)4zuTMJLWBOFWo$!KIu=N7)%ddPFt=lRl25Yuh;%br!|?;ZCtx(+?X+L zoVBG}cFc+M?Tg)qGu?;NJCEEtk=}GXeReQy+_+#2e1bmcxn0?+XJ0us-ZvpEF{X(l zsn%5OOxMiUXAY#>4&K_GJ~)sbIGZ+~TVT%LbygSfwK?N_GQFceKRCM;z ze9r>Y2Xmw?O*4mQcBjo-7np51*7D~EduYnifRWuwIKh-5rJagT#J<1h2M51*@Q$%_ zslxt$%V5)0u1NSGzSXqfPW{SmIzVfGRkLn?kM>tRI;bl~AB5L~QJ>Gt5U0_Kr4U=` z$VXH}5-fuEAd3LO!A#bb(D0~T}@#0V6Cv5ulhg->BPUpMuiLThg@Lj`MIOqAoLCN+K4$vHlD%o)qEJg{w zNka_a2yq@Wf;BDd%ezke4m2ocpKmA*;W)wP6OsFhZ($~5CSiv1p6IONHmv;%WQrcN ze_*U4>RlpQ)Z#QY!%d=KfQbk%yM!I6rV_40*ixPebz&RFrAv&5*rs{Th|eCULW>ua zXYd~H?`lwfKmSn^i9f-IQjr)R4-31+n=k-fm=d0HSxeLOXN-Yn{^X$Oraw>> zA5-p+DfVN^_AzDsm@@y4YP@UlOdPm&c=B+{leIK1S~h1ao3oZJiR literal 0 HcmV?d00001 diff --git a/ops/offline-kit/build_offline_kit.py b/ops/offline-kit/build_offline_kit.py index 53b63a423..a37a5eb87 100644 --- a/ops/offline-kit/build_offline_kit.py +++ b/ops/offline-kit/build_offline_kit.py @@ -184,6 +184,45 @@ def copy_plugins_and_assets(staging_dir: Path) -> None: copy_if_exists(REPO_ROOT / "docs" / "24_OFFLINE_KIT.md", docs_dir / "24_OFFLINE_KIT.md") copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-collector.md", docs_dir / "telemetry-collector.md") copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-storage.md", docs_dir / "telemetry-storage.md") + copy_if_exists(REPO_ROOT / "docs" / "airgap" / "mirror-bundles.md", docs_dir / "mirror-bundles.md") + + +def copy_cli_and_taskrunner_assets(release_dir: Path, staging_dir: Path) -> None: + """Bundle CLI binaries, task pack docs, and Task Runner samples when available.""" + cli_src = release_dir / "cli" + if cli_src.exists(): + copy_if_exists(cli_src, staging_dir / "cli") + + taskrunner_bootstrap = staging_dir / "bootstrap" / "task-runner" + taskrunner_bootstrap.mkdir(parents=True, exist_ok=True) + copy_if_exists(REPO_ROOT / "etc" / "task-runner.yaml.sample", taskrunner_bootstrap / "task-runner.yaml.sample") + + docs_dir = staging_dir / "docs" + copy_if_exists(REPO_ROOT / "docs" / "task-packs", docs_dir / "task-packs") + copy_if_exists(REPO_ROOT / "docs" / "modules" / "taskrunner", docs_dir / "modules" / "taskrunner") + + +def copy_orchestrator_assets(release_dir: Path, staging_dir: Path) -> None: + """Copy orchestrator service, worker SDK, postgres snapshot, and dashboards when present.""" + mapping = { + release_dir / "orchestrator" / "service": staging_dir / "orchestrator" / "service", + release_dir / "orchestrator" / "worker-sdk": staging_dir / "orchestrator" / "worker-sdk", + release_dir / "orchestrator" / "postgres": staging_dir / "orchestrator" / "postgres", + release_dir / "orchestrator" / "dashboards": staging_dir / "orchestrator" / "dashboards", + } + for src, dest in mapping.items(): + copy_if_exists(src, dest) + + +def copy_export_and_notifier_assets(release_dir: Path, staging_dir: Path) -> None: + """Copy Export Center and Notifier offline bundles and tooling when present.""" + copy_if_exists(release_dir / "export-center", staging_dir / "export-center") + copy_if_exists(release_dir / "notifier", staging_dir / "notifier") + + +def copy_surface_secrets(release_dir: Path, staging_dir: Path) -> None: + """Include Surface.Secrets bundles and manifests if present.""" + copy_if_exists(release_dir / "surface-secrets", staging_dir / "surface-secrets") def copy_bootstrap_configs(staging_dir: Path) -> None: @@ -267,7 +306,21 @@ def scan_files(staging_dir: Path, exclude: Optional[set[str]] = None) -> list[Or ) ) ) - return entries + return entries + + +def copy_container_bundles(release_dir: Path, staging_dir: Path) -> None: + """Copy container air-gap bundles if present in the release directory.""" + candidates = [release_dir / "containers", release_dir / "images"] + target_dir = staging_dir / "containers" + for root in candidates: + if not root.exists(): + continue + for bundle in sorted(root.glob("**/*")): + if bundle.is_file() and bundle.suffix in {".gz", ".tar", ".tgz"}: + target_path = target_dir / bundle.relative_to(root) + target_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(bundle, target_path) def write_offline_manifest( @@ -372,11 +425,16 @@ def build_offline_kit(args: argparse.Namespace) -> MutableMapping[str, Any]: if isinstance(checksums, Mapping): release_manifest_sha = checksums.get("sha256") - copy_release_manifests(release_dir, staging_dir) + copy_release_manifests(release_dir, staging_dir) copy_component_artifacts(manifest_data, release_dir, staging_dir) copy_collections(manifest_data, release_dir, staging_dir) copy_plugins_and_assets(staging_dir) copy_bootstrap_configs(staging_dir) + copy_cli_and_taskrunner_assets(release_dir, staging_dir) + copy_container_bundles(release_dir, staging_dir) + copy_orchestrator_assets(release_dir, staging_dir) + copy_export_and_notifier_assets(release_dir, staging_dir) + copy_surface_secrets(release_dir, staging_dir) copy_third_party_licenses(staging_dir) package_telemetry_bundle(staging_dir) diff --git a/ops/offline-kit/test_build_offline_kit.py b/ops/offline-kit/test_build_offline_kit.py index a1daa458f..253f3bdb3 100644 --- a/ops/offline-kit/test_build_offline_kit.py +++ b/ops/offline-kit/test_build_offline_kit.py @@ -40,8 +40,42 @@ class OfflineKitBuilderTests(unittest.TestCase): json.dump(payload, handle, indent=2) handle.write("\n") - def _create_sample_release(self) -> None: - self.release_dir.mkdir(parents=True, exist_ok=True) + def _create_sample_release(self) -> None: + self.release_dir.mkdir(parents=True, exist_ok=True) + + cli_archive = self.release_dir / "cli" / "stellaops-cli-linux-x64.tar.gz" + cli_archive.parent.mkdir(parents=True, exist_ok=True) + cli_archive.write_bytes(b"cli-bytes") + compute_sha256(cli_archive) + + container_bundle = self.release_dir / "containers" / "stellaops-containers.tar.gz" + container_bundle.parent.mkdir(parents=True, exist_ok=True) + container_bundle.write_bytes(b"container-bundle") + compute_sha256(container_bundle) + + orchestrator_service = self.release_dir / "orchestrator" / "service" / "orchestrator-service.tar.gz" + orchestrator_service.parent.mkdir(parents=True, exist_ok=True) + orchestrator_service.write_bytes(b"orch-service") + compute_sha256(orchestrator_service) + + orchestrator_dash = self.release_dir / "orchestrator" / "dashboards" / "dash.json" + orchestrator_dash.parent.mkdir(parents=True, exist_ok=True) + orchestrator_dash.write_text("{}\n", encoding="utf-8") + + export_bundle = self.release_dir / "export-center" / "export-offline-bundle.tar.gz" + export_bundle.parent.mkdir(parents=True, exist_ok=True) + export_bundle.write_bytes(b"export") + compute_sha256(export_bundle) + + notifier_pack = self.release_dir / "notifier" / "notifier-offline-pack.tar.gz" + notifier_pack.parent.mkdir(parents=True, exist_ok=True) + notifier_pack.write_bytes(b"notifier") + compute_sha256(notifier_pack) + + secrets_bundle = self.release_dir / "surface-secrets" / "secrets-bundle.tar.gz" + secrets_bundle.parent.mkdir(parents=True, exist_ok=True) + secrets_bundle.write_bytes(b"secrets") + compute_sha256(secrets_bundle) sbom_path = self.release_dir / "artifacts/sboms/sample.cyclonedx.json" sbom_path.parent.mkdir(parents=True, exist_ok=True) @@ -114,10 +148,10 @@ class OfflineKitBuilderTests(unittest.TestCase): encoding="utf-8", ) - manifest = OrderedDict( - ( - ( - "release", + manifest = OrderedDict( + ( + ( + "release", OrderedDict( ( ("version", "1.0.0"), @@ -216,9 +250,9 @@ class OfflineKitBuilderTests(unittest.TestCase): ), ) ) - write_manifest(manifest, self.release_dir) - - def test_build_offline_kit(self) -> None: + write_manifest(manifest, self.release_dir) + + def test_build_offline_kit(self) -> None: args = argparse.Namespace( version="2025.10.0", channel="edge", @@ -242,10 +276,34 @@ class OfflineKitBuilderTests(unittest.TestCase): self.assertTrue((bootstrap_notify / "notify.yaml").exists()) self.assertTrue((bootstrap_notify / "notify-web.secret.example").exists()) + taskrunner_bootstrap = self.staging_dir / "bootstrap" / "task-runner" + self.assertTrue((taskrunner_bootstrap / "task-runner.yaml.sample").exists()) + + docs_taskpacks = self.staging_dir / "docs" / "task-packs" + self.assertTrue(docs_taskpacks.exists()) + self.assertTrue((self.staging_dir / "docs" / "mirror-bundles.md").exists()) + + containers_dir = self.staging_dir / "containers" + self.assertTrue((containers_dir / "stellaops-containers.tar.gz").exists()) + + orchestrator_dir = self.staging_dir / "orchestrator" + self.assertTrue((orchestrator_dir / "service" / "orchestrator-service.tar.gz").exists()) + self.assertTrue((orchestrator_dir / "dashboards" / "dash.json").exists()) + + export_dir = self.staging_dir / "export-center" + self.assertTrue((export_dir / "export-offline-bundle.tar.gz").exists()) + + notifier_dir = self.staging_dir / "notifier" + self.assertTrue((notifier_dir / "notifier-offline-pack.tar.gz").exists()) + + secrets_dir = self.staging_dir / "surface-secrets" + self.assertTrue((secrets_dir / "secrets-bundle.tar.gz").exists()) + with offline_manifest.open("r", encoding="utf-8") as handle: manifest_data = json.load(handle) artifacts = manifest_data["artifacts"] self.assertTrue(any(item["name"].startswith("sboms/") for item in artifacts)) + self.assertTrue(any(item["name"].startswith("cli/") for item in artifacts)) metadata_path = Path(result["metadataPath"]) data = json.loads(metadata_path.read_text(encoding="utf-8")) @@ -258,6 +316,11 @@ class OfflineKitBuilderTests(unittest.TestCase): self.assertTrue(any(name.startswith("sboms/sample-") for name in members)) self.assertIn("bootstrap/notify/notify.yaml", members) self.assertIn("bootstrap/notify/notify-web.secret.example", members) + self.assertIn("containers/stellaops-containers.tar.gz", members) + self.assertIn("orchestrator/service/orchestrator-service.tar.gz", members) + self.assertIn("export-center/export-offline-bundle.tar.gz", members) + self.assertIn("notifier/notifier-offline-pack.tar.gz", members) + self.assertIn("surface-secrets/secrets-bundle.tar.gz", members) if __name__ == "__main__": diff --git a/out/telemetry/telemetry-offline-bundle.tar.gz b/out/telemetry/telemetry-offline-bundle.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..e000162b09f065221c5150582c540b5e1d102402 GIT binary patch literal 10988 zcmV12a(OLpW@c<@Ze=ZEb#7#AWiE7Kascgp+j84T zvS2?mg8xt^&R{0-2*&}II$yW{=0AUX&I z|N4)^Loc{(B;4b3o?H6!K1qUzb>o;PakWyd3}-y%%%;Ct z7zH;^tdW}G>EYA?Km^+WC#0daj8T7SY>#Q(8yBkr*AZM8Bw?Y-$=4r=|UgW5r} z2Ba{mGiZfPbVYjtZP_t367$IdtGfO*j}sOKuAi_e6iOm{!sFd)1)Fsz@h-D{hk3zu z%=p|*5^9bOfUrILkKg~ZAI1PHx^baW9Xs~u5uoZP+!y`)$ey_#XCK++A_@cO@<)7E zgNKQK=oh{l1pu?Av4vm$)V;3X+H-FNWuMwH2U0QnBT6hHJHesbo-dMx?XltcV3+lW z=a(=(6Yk#d2>Ngt1<;=vUqCNDgh7<>Xs0gQjRBT@%^RUEqZ>7<#goVmd9KZ5;lp^) zuw_RYhD?qMgQ2itF!HS>d>#h6p&|a21ODDkW^4|W;1Y<*?JoO!_gu`!B$$UVomdPg zD1ljAYQPKLxf=dC{E-6>Fvl>U*G3Zw&a)&5jX}h5GV#wa@Z5>Z3%Wu*Y&=fKjYl;J zF@On(T!+UrOroFI1HuSU;f=HbKro2{Pc%TJbjokI7ld;tz}^SZyQvqv$4{*~d(M51 z^KZ(a5L<)@axVi?4kjDhP}6q+ z`;_>0#klR#pzUCrq62qtw;*^ThyVFUOXx*cOqFRu6A^l*e>m>ouK`j4f>^cf3`VXdMj1Fto z>3^O5z1C6sJu#}{gL||!1tqJ){@bsEVekCB3wof{hhG**5!OlXYS{UoD?TtkK#?l> z&4dUdz&e00dp{P!gZG8tgyd~a?6p{Si*f80f|O#2r9h`*N&)z*-rruIUG=P!^RwQ? zP>e?sEx3j_OOkNY1RR$W@HHl&T7mTd@^~D`m0~9#X^$5mZ%VbYme|*FxP`)-4t@;> zsv{Nby6dC}ijo~Pl>lbxdlI&@7!G?ET|nyWw6pNv`N4ai`Z>0f$xJqc8iz$Mtl0ep zs)HD{FFxT8A1r`U4oCq|riHizUqPq11_8N41%VTcgV;>~T|bECwpSreAoUATCZ)Tj z5xl(Wzv>NN^j;6F-o@!<|LkJeQN;OV|8V~|2>>*D(9S~cTT)|z(Sd=cui}jbIJ!as z0j%d0MbMQmNz0WlM#hydO2Lv7<9ZkNmvsX_?7h0|r^Ahv;!dLxljRGv;+lZ4*KRfi zny7R~K*}_<*T1;o(%=@MFPagETF=hTdz}VPCOL7T0Z*alR4viX!#$^r(ruc%!QBzY_Ft*PyL&_P)T89$ImtT!j1X)l>s zlNq19lbgsI=KiABJs-YUX6&uK*4`fNMj>n!)6IY`VVihX00(pg+G?dmngD6VM1n;N znBG`Dw5S#|aNd7zo%i1K&O5F@4G1-(=jo#AZ&0QJ*rAnWQ5Cj=Y;KqSRL$)K`uNYb)qf9PsDEE<_a=_OO zsrLnuCPuVLLA1x9D8yqqxMLe}XWz3iw<8W81DFgJdeF~V;5kNM$tQ$(C~ZV~r6;zh z9qA$TP={XJIk+P%io`(R3dU)a>wZHoI4KzGxq0R`V)B(nyEp`??-95G-f=F%g~-I&2Q_u3CcY@Fh!JM~Hh176k!xZK#D>tOhg z94m`BCWg2$dkQR$amP&rtG#8x6tM<;1=%D)1$5y>#6=8RHgwAz)0Adn_&J9z!N5Q} z31A3pZGhGjVDuGWk52>O5c#MnVSHz4z*`+k8bc2ILQEVM#%6(|m6?NNW+z6l#@zK~ zP+u1&CcuUf-0KjBfhN;ffwG@4??4kU*9Ln@gLl%f5E;e;9Qs)hNdxH&?FBcu%i&_6 zsI55R6h_F|J&E|w8&dM;(Es+A^}lwh|2>HQcU+|ZwGNJt4@(8`Th{+_a_3jn|61++ zRzCitdAL{Ve-H6od%@WD$igfl#|j-gE7^-XN1F?L2 z&O2-?^;=~6bxT_4Wv+1g;ktZmnY{N}e zE%H<4!Dt^vm`YuJC;yC_C66@w*uuwW8*l9_W6RBA;jfi6%AIkA}oWFoIQ8SC$|$%{s~nvT(+8v@3QP0+x4XD%lako4Fvyp z{{k?>cQu>PAa9f9Iv%$|5bgs9!?#EQn%f_&u@_9xM;$MD{2?jKtRM~8@)JWeZ@ zN_x0fVk0VQ11D5IJf2}arBum7qk5@bXyR*{~?xU}~{I{IoHcib>PD zs-rEx79F7_^RFIo7;hSG(neuA1bR48)*{AnXw$Yu0b}q^23)H4l%C4+gt*32^X`M=pb%*X#6whv4B{~;caYW21R9gs!hc61mFR~h z`(d0=yWz6+LICth6rR}7DGOydAynoHS@d0+hG%xfouc8XR-%QHpiR+H8Ld}Ji-p)B zEb1NpP_BYx9*R&M{Z$?`vqLu}ZQ<%SM^wfL$fIb|bD?eR5vLgsJm?n1`^2eEwfxOO zff4&AMQbZlKb8O6druCIO0Mv2+y7VFE55A!-#lvO^MCB^AMKa&|3f^FbQrS;-Ykm4 zu2v{lVvMo=>Ok^!WKV71u7kwipjb#@q4O{USJKFYA2%KenX8t0i$G<>t{sNntq9pQ zP6gD30IZ?|9DbA98ih43<^a(VQWXkJ$|^@TR5a#MDPf#cW<6aX=5{ohEthLIpB%Lg zP`ol*$B<^ezWw(4_D#5!rH&R`>tP$?UYWtUmaO}w>h#Wcv6RZ{m$V9VF5|B2E;tl>&Z6UZt zifC3VP9kp4O_!ZAv}oUG>UWhc(xSb-0G0N7RQ5Nc(rT{6WpD3rRwN%3;Bl}Pjn;|= zj@q-Fv{3H<<^I34|2Ovz4qJQW{{QXl|1<8*(}arn*M_H{KaUw zOxpKpN&6lqXWrZDpD0n^gLSzSoBw5=QvWaY|7HFEc>hWBxLoGntp0CY_Vk6;|7!ie z1*-p2{=fbG(*OS;kCFc&0;$c8d8!A94DnksSHomx`#wr;>c!ml-6@Zg!OU(S9IE#y zmyyIP{&eSQq-VJByi2^B4|I#O=|zxSMjVtD(tG5YBlar^lc@knfx9XtKqZq0JI$vp zd4Ks=R- zuu{IVL+Uu)@D|l{5V163inbj)u@x5ig2eke?4$0;85KCy(a7c*N|RdCXYTs$d@(=Q zRC4Bv#P6=m`Yvq%Dw|ohc2T-_6V;CPzi9yV_-0aPe`2yj* z;)zY;vD}`$a9qj(mqid~Jaw9do}FOtat+uH-ICB*8s83n%M^YOUEr%q)5k1U4vg=r z;`A{yL%4c{jN4)VUeg&%Yhqm+MPtsLlh~eKF2e>iFld{ql| zmAcOCFP>L{e;s6pPv#~BS{NjBR)Wp3kTZrErC*))QB$u684C|hjgdt!^|*jQVtEs< zFffT%1;t+S+k%o7tmUH8!w9IWpm>7sbqspEL_bA!=zRsZ4B8WF+z0iN*XV45F!?O4 z($r*d**`x!`O)fKJU_eWSyuOCc=o0{JnLUr*0cWANpJY~{$=lK(7))S;o?pA9GEa| z7+d!Nb=rH@eSJQZhQYy?0or?YHtfDWJsY0)pAP^6QrA{R8Bhw@R>$1d)Un;VSba5E ze)S-_HpI6juJ@jgJMCus0QS6kv&Nn4;(b%{&S2puda>7-I1sUdlU59R7JPSe(K$9? zY+i-DSC@_#V^MZ>G49_y!_zz(WwG~O@u|7yR`f%ZjTInO<7_Z^-Mc!4X~%R$KU&sF z|Ki!%^E;rtRI~ef_@aMxHiVx~yMx~AtMdZp-8tUhZ?1wc==Dxd2Hj_uub-Zu4PF2Z zYPz8<7Is8q!eshU9*LduxOvXdu-?SKLsfz%Ev=20j8x+N+hzz0C0)hx!M-K>a1T$tE1s^L}6Ji6!V zTdIDil{W`*+DYfo2@;a=$*bk?_F8e-(L4# z0~30Ac5&Ka7j_b_EOOBu4lIi()Q56Fp!CmvyWL){GXUc3KJU@exByZG?lu@+on1UP z_p)>}mUp!xSf|}#7gvdqZotwomh1h!fAtdoy?PBOAe@t~_<=CGr>DTR2V%7mE~vsp z#13J(q5Nn&H*Sn4bG%f( z>o|C8*SgNE1M@e!D#+3T>}8r5Kxgo|Vo$bd?10t#ZcNMdYRuL1|=^&CaxyH~p6b5O54tRpag|?bX^#!FfB>ySwu>e59vMCovxiJ2j z7Dnm+Dg8fX{Ac<6YxsZgFdqf*=VCxV%_FqQ|FhRR%*TH=kIMWn5Ayu_YmK>6R(~Tr ze%h2>Pv=}+_4C_r)ruXuZD++=H0c;6KTe-aBQ1fP@3&Y+X zL)9&f%+}5h`^bDuY~m+upU}|VF#@1Fc>mCyJ0>b3Rv%dgbgfp)j;S4@_f-$($0A8W}T#aw#!ly@?j%+9l!3}JNZ?-pYo`5bs#-3=P+ z#!(H}NA`~e^i?)f42SMU@;Q14w+b@dD}LJ1;bQ4jlGQ7V=a<9u6L3{okQ8w3&^4w; zc#YMkrFQkaO2e?QMwoH zA5urF!3eg@^1=?AhoE}ug*EhbR&!pooV7L^M85<9b==Jmj*B5k2kM6U=vghUFYK)w zs#9cXaeWHf+*BPT1+xem0*axni0aM5m$B3v2#K8E(m8L^Q=yxF6#=cq?Pc%BPe)Bg z1gysl3Vjl0C*98&GkZN=!tZ}?v%AiB7?|ZlT9NZ>0ute^JNlTk4R!i3jgA9ppa>r$sUM&$!}# z$$nkoddn8ddEoY%+4DOYr{tflF9&P8(mFsLp(bumUC#hy2vd%n)6gbTHj>~+ATv{o zXtk*ZHo|?!s4W&kk;Ju3|EcFb+ndkUG1ij0aJ%Q4G-KxOu=%9LVMrtwbqt+jb#XV= z<+t<@ZqS1f1Vz*>F+UM!mzkqG!Aa+Fe`y|Hqm$|Igu*qochiW&Zzf$p5qHs`t-23bY~r z$KFA6>HgQlW_kbDgFNXmpS}8%E#n}!N(hp6L+tFOBAvzm#8Ow%V6 z*kIF21=d*Z-?&lWqu)eBS(O}s6@etnRqDP;Q!}i{3UCJ?=~0>DL;$NnmfFStq@S8>hh6jNf@4&DH_qH^|jvDM?CZ`@r2M_(+q z2~5#+EZ-(}-$^;|zdTzz7?>KbgBNIsBzd6UHEvpsa1qanVSWDCXfU5OG$}Io7@52g zqQ>fVBh%#S^irJ-%e)f>D`27tKklr+O;^kMX@I|=x<%=Ww(!Lx)*qf-|17^S^1i@}CU$u}fy z!?hyJS7v??bU?M7ougw%D%A>Dgb8%S;z}p)e zuoZB%YUl*pEC9r5G?ZCx>uq|oO6E3l<+!w?B+c%4pg&}C;3)eV-OB;>apUuOg7K)C z3*+Pf=C>_PaR2lwXQ}^{`d_L4mHOWX{jVm|LVu0?&&b^K@qg|8GXC$+p#Lo;ev`T% zzHb;GZ`$>Pz0K;Lc}436Rj-bbMMB?8YZ*UHeQ$%HsZyHfDt9&j&q$f+H;VV~H1b_p zCyHjSBVH}MDm?K_Xucx7Q#Fh9TSFB*>7Sp|$x&q_RXQ@;+gGb^ZP5 zBdymdgtmF+)?J#@xyOaTcT#XxQVOjqSv^#hQs7LQFDgvxRdk z^xS%D^Arn`x$_K*-16#f()-P?uGT34b@$0FxDlaq_sjM^b7~1cs zTdkNwl;>r7y{3Pr&3v<(QR)9H^}laZ|9i5xzjx4nQZC?cLI1-GTjl{~SBOp9Y{*60 zWz&T*cfN-Iuhlv{%;o=Uw_E!MrT+I2&m%Telg@N-m~rliJzs$6f+8;F}p?#K`eG0*!}VQe*%KL7^v)mAjFxQ#PY%&ohCaLy$mBB+IWwl&w}X`J5j^9)mt`R z_>RZf6q>NzsCI3KAv9u0*fI5jLj9_hM~_&m&iXf~fAjaEdpym42xAq^T^~I{2HDQ;J;l^KJg|Y zaRWFWef!92@?rX?_?7;xUt{qRxmQLEKBvwVIh-HaT(l!1;F*!7U!-et*!b2Mh?A}X z^Eib`UYO5@i7GL{G~6{nT6V}&paYB{oh(C^Ajn@af^w$w=I*tDR}uo`(_xNlU;9A} zqaWW1gdiJ5@|pX$C^8(VB4MMYd*4=j<3>a&b2o}`A>(>_!M(YN;KwCFuSQx07-6Q+ zD9KTFyDqXzsP%-{Y@Y}-;|`V?jYeZTp5bL6hm%{)+ z>J2=*3^l|NX4SK63Cn^n7R3S6C0m5k0KG-jffOR@Yn}}X%*0_?^Jujqv|lFwX0<6> z8!^KcA*(r}wLP;nsZ*-9*mg>6yW%F+UAmBM=RQqljI?Ic`*n7;5c3qzgLj_4g6U$RxJWBX07VZQ# zk1=P9STH8hl;-0A_dU07)VRUl*N6h(Jm8%DG)gx$nJC8C!AEbm6;v4}NjyMMEC$K=g z+q$|&vXkNE7+60^JO*td6xld2WDs0db&mZ9j(e?!f=!J~f3kBwWMZ<4ivgxTWTLK? zi-`e||LAEWnv7t*a=FW;Ly{8pu3D2~G{oXzb34Q|z5(X;g>~!QW=8>&YXCdGj%=Wc zTFv#Lm!6k)pA{zIAWpv~J3C&ODbJnB9=r}oP?g*TEp}dDJH$OGKdUAMCAG%Hg_f9+ zDjGe|T$Q`b5Z<7juCB=732<~Ec6S=e){F5>8 zsy$a01(fU8zhw75&__a`b~~Ag!|yO?1%lUAuY3Bc2fTTN-&3U|%b2(CFR>9pgTi*Z zgMjHTm4`r*kmjjI1SX;&Bn>T_jP4U8JlawHSF;$;T%cxRS2i1g^o8l-M?xr$7jxiC z0bZwNPw@Lm5WCl~>A-*NWD(&ybAX4{YWPMGuHdhulm6iB`2_{GUG<)g8tTQ_Y42i) z=~jpRm%WP-DqOIqz^dMc?k*$+j?mGFKrxhQ#HOBoEp*CL%wIMu0ID{w0&Z>#)dH`-nnSTiF1w5%i{wh`tiSq13)4rb@mt zBsih+#dnki0s0<7AEGD)Z=;BNG*Yqw?#DoPsq6*g6x!D~2_ureEY}fqIMNrojg0Hy z{s&CjR4}v=8P@{SIFkJf7d@>(kU?RTk#B{zj~MPFh9`!67zK%mS$gNfoQM|0>0GjX>6;C=>;hj&XyVg2-4WVE55k1b9+^QUyhTxzm(kZK=65ghbvT-6gR< z1b#5TH9)oYfH$4@x`Up~1>VC1xz^=*_s6IG{!0x`UcMpLss}7c@MbO7+cC(>KpCa~ zr}Y1n{-5%BIR8&-JoudTzsCP_u(#h{I{(!!?|*u5|If1RXjzSG?9M=9=H}nzU+lLQ8@r&_7{c+&ST*uk*OCL`3J;V^_;T=N*T_NkC}h; zFS7I-Bk+-7pdI|9MBeV@8Eb&jfg%=&UfPX;VSw_?2uhgJFOf>=1Xv6rJs%l{b|G#g z?Mmm!_XJ-$6SfW6PbbCL5s+y3HKZ0nIuEYz2`BH!0?FpR9XUBS7IY^+`%4~}oMGmV z-~S&-{S8SzBkh+HQ%%Ub(vXSL6p#_f#db8AeWG*X9v+5e=RxI>wEvsF3q{jpKK?ZC z!sj^%zM$d%4xp4X)sl!<)NcTGhq_{+TY8~egcl<6A7tN4hjsY?5&8)mH!WF`V%-W# z(NYmbF#fHm1S|n0hDzlmqm-aoF(&UCTDO-K8x=x0s1Gn(R%j&)$5Qnk=h5|Tm} z+|%qn_7*&hTw!$={F@cHl=QNY>isCA+lnro~6 z2f*T)XJ4DKkMH?dX05Gx?vzh%C!VzUhlomyTf-X)h}4w|yaUh@U84?E@tlGzZO3ti zNo7FXNYFq$5Sfe$!`y3vZ^_H7BU_ZsBq+{T2i&;yv5CcJZ{Hg2P`ZDn}S8SU~h^3`2|E8=# zv>a~%>A4tqm~nt3#{O9l4e_GMbt4)YItK*Ku8X1r&l(Mqpy7a_fP3t>>t+A!jqN#{wkX=GX( zF952pBQ{|CI$wB+OMRN$_VSZM6#Zs7<=#q){%k6h=q-)$M(`pYdsP$CX#eODZPDbG z7zaUu3@%h*>G3+pmgbu|W>%{?u`~Vpc79&M#iEWQG!v-$*DY-Y*>3^jIp%z)s zHIQF(nI@i^i+_&#e17qd)Vd$-m0o$j za>7#t;w45Jd>X34vOzW&`1=#E6B=+KdNRW7z9)~B|fZ4dT#+4GaZF6;h+pPzccct`li=df!c;qco4z^00! zs!M=#;TmbU7e=U3+z}CG>W!5_A&Z#&;`a+F62SxR996zm!Ay$Y)BA;&yy{B<9;iu* z+4zaQWhbU5p8^HAh~H4FOT-E|byZIi^zjj8!{bG~=VYLWHjY?GMiTjH4Wehhwr% zmKL6P(f!qa2o^0o5jGb~+5uSGynbDm>sk5m^j*284^P~)X-ZQFPRb2CS;OKHuJlcj z8fsl@+qJ4DWCEwpYkc6@+qJ4DWCEwpYkc6 a@+qJ4DWCEwpYr*Zp8pR;*M(RB7y$r+gJ#kI literal 0 HcmV?d00001 diff --git a/out/telemetry/telemetry-offline-bundle.tar.gz.sha256 b/out/telemetry/telemetry-offline-bundle.tar.gz.sha256 new file mode 100644 index 000000000..2c189c9e2 --- /dev/null +++ b/out/telemetry/telemetry-offline-bundle.tar.gz.sha256 @@ -0,0 +1 @@ +dc3938d79d4e0b9a77e92dc6660391f36230b8d16c9b24b7164b6a1e6723666b telemetry-offline-bundle.tar.gz diff --git a/src/Api/StellaOps.Api.OpenApi/tasks.md b/src/Api/StellaOps.Api.OpenApi/tasks.md index f6baf1d94..a10461b50 100644 --- a/src/Api/StellaOps.Api.OpenApi/tasks.md +++ b/src/Api/StellaOps.Api.OpenApi/tasks.md @@ -7,4 +7,4 @@ | OAS-62-001 | DOING | Populate request/response examples for top 50 endpoints, including standard error envelope. | | OAS-62-002 | TODO | Add custom lint rules enforcing pagination, idempotency headers, naming conventions, and example coverage. | | OAS-63-001 | TODO | Implement compatibility diff tooling comparing previous release specs; classify breaking vs additive changes. | -| OAS-63-002 | TODO | Add `/.well-known/openapi` discovery endpoint schema metadata (extensions, version info). | +| OAS-63-002 | DONE (2025-11-24) | Discovery endpoint metadata and schema extensions added; composed spec exports `/.well-known/openapi` entry. | diff --git a/src/Graph/StellaOps.Graph.Api/Program.cs b/src/Graph/StellaOps.Graph.Api/Program.cs index 96eaa0c29..ccda25197 100644 --- a/src/Graph/StellaOps.Graph.Api/Program.cs +++ b/src/Graph/StellaOps.Graph.Api/Program.cs @@ -60,7 +60,9 @@ app.MapPost("/graph/search", async (HttpContext context, GraphSearchRequest requ return Results.Empty; } - await foreach (var line in service.SearchAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.SearchAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -113,7 +115,9 @@ app.MapPost("/graph/query", async (HttpContext context, GraphQueryRequest reques return Results.Empty; } - await foreach (var line in service.QueryAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.QueryAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -166,7 +170,9 @@ app.MapPost("/graph/paths", async (HttpContext context, GraphPathRequest request return Results.Empty; } - await foreach (var line in service.FindPathsAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.FindPathsAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -219,7 +225,9 @@ app.MapPost("/graph/diff", async (HttpContext context, GraphDiffRequest request, return Results.Empty; } - await foreach (var line in service.DiffAsync(tenant!, request, ct)) + var tenantId = tenant!; + + await foreach (var line in service.DiffAsync(tenantId, request, ct)) { await context.Response.WriteAsync(line, ct); await context.Response.WriteAsync("\n", ct); @@ -272,7 +280,8 @@ app.MapPost("/graph/export", async (HttpContext context, GraphExportRequest requ return Results.Empty; } - var job = await service.StartExportAsync(tenant!, request, ct); + var tenantId = tenant!; + var job = await service.StartExportAsync(tenantId, request, ct); var manifest = new { jobId = job.JobId, diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs index c4a403067..f365c4013 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphDiffService.cs @@ -8,14 +8,16 @@ namespace StellaOps.Graph.Api.Services; public sealed class InMemoryGraphDiffService : IGraphDiffService { private readonly InMemoryGraphRepository _repository; + private readonly IGraphMetrics _metrics; private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - public InMemoryGraphDiffService(InMemoryGraphRepository repository) + public InMemoryGraphDiffService(InMemoryGraphRepository repository, IGraphMetrics metrics) { _repository = repository; + _metrics = metrics; } public async IAsyncEnumerable DiffAsync(string tenant, GraphDiffRequest request, [EnumeratorCancellation] CancellationToken ct = default) @@ -26,6 +28,7 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService var edgeBudgetRemaining = budget.Edges ?? 10000; var budgetRemaining = tileBudgetLimit; var seq = 0; + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var snapA = _repository.GetSnapshot(tenant, request.SnapshotA); var snapB = _repository.GetSnapshot(tenant, request.SnapshotB); @@ -39,6 +42,8 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService Details = new { request.SnapshotA, request.SnapshotB } }; yield return JsonSerializer.Serialize(new TileEnvelope("error", seq++, error, Cost(tileBudgetLimit, budgetRemaining)), Options); + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/diff")); yield break; } @@ -49,15 +54,15 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService foreach (var added in nodesB.Values.Where(n => !nodesA.ContainsKey(n.Id)).OrderBy(n => n.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } - yield return JsonSerializer.Serialize(new TileEnvelope("node_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options); - } + if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); yield return tile!; yield break; } + yield return JsonSerializer.Serialize(new TileEnvelope("node_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options); + } foreach (var removed in nodesA.Values.Where(n => !nodesB.ContainsKey(n.Id)).OrderBy(n => n.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } - yield return JsonSerializer.Serialize(new TileEnvelope("node_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options); - } + if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); yield return tile!; yield break; } + yield return JsonSerializer.Serialize(new TileEnvelope("node_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options); + } foreach (var common in nodesA.Keys.Intersect(nodesB.Keys, StringComparer.Ordinal).OrderBy(k => k, StringComparer.Ordinal)) { @@ -65,7 +70,7 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService var b = nodesB[common]; if (!AttributesEqual(a.Attributes, b.Attributes)) { - if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } + if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); yield return tile!; yield break; } var diff = new DiffTile { EntityType = "node", @@ -82,13 +87,13 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService { foreach (var added in edgesB.Values.Where(e => !edgesA.ContainsKey(e.Id)).OrderBy(e => e.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } + if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); yield return tile!; yield break; } yield return JsonSerializer.Serialize(new TileEnvelope("edge_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options); } foreach (var removed in edgesA.Values.Where(e => !edgesB.ContainsKey(e.Id)).OrderBy(e => e.Id, StringComparer.Ordinal)) { - if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } + if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); yield return tile!; yield break; } yield return JsonSerializer.Serialize(new TileEnvelope("edge_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options); } @@ -98,12 +103,12 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService var b = edgesB[common]; if (!AttributesEqual(a.Attributes, b.Attributes)) { - if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; } - var diff = new DiffTile - { - EntityType = "edge", - ChangeType = "changed", - Id = common, + if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); yield return tile!; yield break; } + var diff = new DiffTile + { + EntityType = "edge", + ChangeType = "changed", + Id = common, Before = a, After = b }; @@ -126,6 +131,9 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService yield return JsonSerializer.Serialize(new TileEnvelope("stats", seq++, stats, Cost(tileBudgetLimit, budgetRemaining)), Options); } + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/diff")); + await Task.CompletedTask; } diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs index 28dcb8231..52e000c4f 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphPathService.cs @@ -9,15 +9,17 @@ public sealed class InMemoryGraphPathService : IGraphPathService { private readonly InMemoryGraphRepository _repository; private readonly IOverlayService _overlayService; + private readonly IGraphMetrics _metrics; private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - public InMemoryGraphPathService(InMemoryGraphRepository repository, IOverlayService overlayService) + public InMemoryGraphPathService(InMemoryGraphRepository repository, IOverlayService overlayService, IGraphMetrics metrics) { _repository = repository; _overlayService = overlayService; + _metrics = metrics; } public async IAsyncEnumerable FindPathsAsync(string tenant, GraphPathRequest request, [EnumeratorCancellation] CancellationToken ct = default) @@ -29,6 +31,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService var edgeBudgetRemaining = budget.Edges ?? 10000; var budgetRemaining = tileBudgetLimit; var seq = 0; + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); var result = FindShortestPath(tenant, request, maxDepth); @@ -42,6 +45,8 @@ public sealed class InMemoryGraphPathService : IGraphPathService }; yield return JsonSerializer.Serialize(new TileEnvelope("error", seq++, error, Cost(tileBudgetLimit, budgetRemaining)), Options); + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/paths")); yield break; } @@ -58,6 +63,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService { if (budgetRemaining <= 0 || nodeBudgetRemaining <= 0) { + _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "tiles")); yield return BudgetExceeded(tileBudgetLimit, budgetRemaining, seq++); yield break; } @@ -75,6 +81,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService { if (budgetRemaining <= 0 || edgeBudgetRemaining <= 0) { + _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "tiles")); yield return BudgetExceeded(tileBudgetLimit, budgetRemaining, seq++); yield break; } @@ -93,6 +100,9 @@ public sealed class InMemoryGraphPathService : IGraphPathService yield return JsonSerializer.Serialize(new TileEnvelope("stats", seq++, stats, Cost(tileBudgetLimit, budgetRemaining)), Options); } + stopwatch.Stop(); + _metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair("route", "/graph/paths")); + await Task.CompletedTask; } diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs index 2b72a6f0c..6ffb5b902 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryGraphQueryService.cs @@ -49,21 +49,17 @@ public sealed class InMemoryGraphQueryService : IGraphQueryService var cursorOffset = CursorCodec.Decode(request.Cursor); var (nodes, edges) = _repository.QueryGraph(tenant, request); + if (nodes.Count > nodeBudgetLimit) + { + _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "nodes")); + yield return BuildBudgetError(cacheKey, "nodes", nodeBudgetLimit, nodes.Count, edges.Count, budget); + yield break; + } + if (request.IncludeEdges && edges.Count > edgeBudgetLimit) { _metrics.BudgetDenied.Add(1, new KeyValuePair("reason", "edges")); - var error = new ErrorResponse - { - Error = "GRAPH_BUDGET_EXCEEDED", - Message = $"Query exceeded edge budget (edges>{edgeBudgetLimit}).", - Details = new { nodes = nodes.Count, edges = edges.Count, budget } - }; - var errorLine = JsonSerializer.Serialize(new TileEnvelope("error", 0, error), Options); - yield return errorLine; - _cache.Set(cacheKey, new[] { errorLine }, new MemoryCacheEntryOptions - { - AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(2) - }); + yield return BuildBudgetError(cacheKey, "edges", edgeBudgetLimit, nodes.Count, edges.Count, budget); yield break; } @@ -206,4 +202,20 @@ public sealed class InMemoryGraphQueryService : IGraphQueryService private static CostBudget Cost(int limit, int remainingBudget) => new(limit, remainingBudget - 1, limit - (remainingBudget - 1)); + + private string BuildBudgetError(string cacheKey, string exceeded, int budgetLimit, int nodesCount, int edgesCount, GraphQueryBudget budget) + { + var error = new ErrorResponse + { + Error = "GRAPH_BUDGET_EXCEEDED", + Message = $"Query exceeded {exceeded} budget ({exceeded}>{budgetLimit}).", + Details = new { nodes = nodesCount, edges = edgesCount, budget } + }; + var errorLine = JsonSerializer.Serialize(new TileEnvelope("error", 0, error), Options); + _cache.Set(cacheKey, new[] { errorLine }, new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(2) + }); + return errorLine; + } } diff --git a/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs b/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs index 09a148a9c..ca52eecc0 100644 --- a/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs +++ b/src/Graph/StellaOps.Graph.Api/Services/InMemoryOverlayService.cs @@ -44,7 +44,7 @@ namespace StellaOps.Graph.Api.Services; } // Always return a fresh copy so we can inject a single explain trace without polluting cache. - var overlays = new Dictionary(cachedBase, StringComparer.Ordinal); + var overlays = new Dictionary(cachedBase!, StringComparer.Ordinal); if (sampleExplain && !explainEmitted) { diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs index 587f4113a..469dd9a9f 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/DiffServiceTests.cs @@ -11,7 +11,8 @@ public class DiffServiceTests public async Task DiffAsync_EmitsAddedRemovedChangedAndStats() { var repo = new InMemoryGraphRepository(); - var service = new InMemoryGraphDiffService(repo); + var metrics = new GraphMetrics(); + var service = new InMemoryGraphDiffService(repo, metrics); var request = new GraphDiffRequest { @@ -37,7 +38,8 @@ public class DiffServiceTests public async Task DiffAsync_WhenSnapshotMissing_ReturnsError() { var repo = new InMemoryGraphRepository(); - var service = new InMemoryGraphDiffService(repo); + var metrics = new GraphMetrics(); + var service = new InMemoryGraphDiffService(repo, metrics); var request = new GraphDiffRequest { diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs index 6f6c1a880..72e789fab 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/PathServiceTests.cs @@ -13,8 +13,9 @@ public class PathServiceTests { var repo = new InMemoryGraphRepository(); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphPathService(repo, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphPathService(repo, overlays, metrics); var request = new GraphPathRequest { @@ -39,8 +40,9 @@ public class PathServiceTests { var repo = new InMemoryGraphRepository(); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphPathService(repo, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphPathService(repo, overlays, metrics); var request = new GraphPathRequest { diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs index 419fae382..d1dcee534 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/QueryServiceTests.cs @@ -107,8 +107,9 @@ namespace StellaOps.Graph.Api.Tests; private static InMemoryGraphQueryService CreateService(InMemoryGraphRepository? repository = null) { - var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - return new InMemoryGraphQueryService(repository ?? new InMemoryGraphRepository(), cache, overlays); + var cache = new MemoryCache(new MemoryCacheOptions()); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + return new InMemoryGraphQueryService(repository ?? new InMemoryGraphRepository(), cache, overlays, metrics); } } diff --git a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs index ddc9bf3a1..464aa2ba8 100644 --- a/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs +++ b/src/Graph/__Tests/StellaOps.Graph.Api.Tests/SearchServiceTests.cs @@ -121,8 +121,9 @@ public class SearchServiceTests }, Array.Empty()); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphQueryService(repo, cache, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphQueryService(repo, cache, overlays, metrics); var request = new GraphQueryRequest { Kinds = new[] { "component" }, @@ -154,8 +155,9 @@ public class SearchServiceTests }); var cache = new MemoryCache(new MemoryCacheOptions()); - var overlays = new InMemoryOverlayService(cache); - var service = new InMemoryGraphQueryService(repo, cache, overlays); + var metrics = new GraphMetrics(); + var overlays = new InMemoryOverlayService(cache, metrics); + var service = new InMemoryGraphQueryService(repo, cache, overlays, metrics); var request = new GraphQueryRequest { Kinds = new[] { "component" }, diff --git a/src/Policy/StellaOps.Policy.min.slnf b/src/Policy/StellaOps.Policy.min.slnf new file mode 100644 index 000000000..9c0b4dbfa --- /dev/null +++ b/src/Policy/StellaOps.Policy.min.slnf @@ -0,0 +1,12 @@ +{ + "solution": { + "path": "StellaOps.Policy.sln", + "projects": [ + "__Libraries/StellaOps.Policy/StellaOps.Policy.csproj", + "__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj", + "../__Libraries/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj", + "../__Libraries/StellaOps.Configuration/StellaOps.Configuration.csproj", + "../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj" + ] + } +} diff --git a/src/Policy/StellaOps.Policy.tests.slnf b/src/Policy/StellaOps.Policy.tests.slnf new file mode 100644 index 000000000..1a7f46a2d --- /dev/null +++ b/src/Policy/StellaOps.Policy.tests.slnf @@ -0,0 +1,9 @@ +{ + "solution": { + "path": "StellaOps.Policy.sln", + "projects": [ + "__Libraries/StellaOps.Policy/StellaOps.Policy.csproj", + "__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj" + ] + } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs index 7709313b5..5ef09dac2 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Surface/SurfaceManifestStageExecutor.cs @@ -10,6 +10,7 @@ using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; using StellaOps.Scanner.Analyzers.Lang; using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Entropy; using StellaOps.Scanner.EntryTrace; using StellaOps.Scanner.EntryTrace.Serialization; using StellaOps.Scanner.Surface.Env; @@ -185,6 +186,41 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor RegisterArtifact: true)); } + if (context.Analysis.TryGet(ScanAnalysisKeys.EntropyReport, out var entropyReport) && entropyReport is not null) + { + var json = JsonSerializer.Serialize(entropyReport, JsonOptions); + payloads.Add(new SurfaceManifestPayload( + ArtifactDocumentType.SurfaceObservation, + ArtifactDocumentFormat.ObservationJson, + Kind: "entropy.report", + MediaType: "application/json", + Content: Encoding.UTF8.GetBytes(json), + View: "entropy", + Metadata: new Dictionary + { + ["imageDigest"] = entropyReport.ImageDigest, + ["layerDigest"] = entropyReport.LayerDigest, + ["opaqueRatio"] = entropyReport.ImageOpaqueRatio.ToString("0.####", CultureInfoInvariant) + })); + } + + if (context.Analysis.TryGet(ScanAnalysisKeys.EntropyLayerSummary, out var entropySummary) && entropySummary is not null) + { + var json = JsonSerializer.Serialize(entropySummary, JsonOptions); + payloads.Add(new SurfaceManifestPayload( + ArtifactDocumentType.SurfaceObservation, + ArtifactDocumentFormat.ObservationJson, + Kind: "entropy.layer-summary", + MediaType: "application/json", + Content: Encoding.UTF8.GetBytes(json), + View: "entropy", + Metadata: new Dictionary + { + ["layerDigest"] = entropySummary.LayerDigest, + ["opaqueRatio"] = entropySummary.OpaqueRatio.ToString("0.####", CultureInfoInvariant) + })); + } + return payloads; } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs index 0e73560ca..8ff2ca248 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs @@ -28,4 +28,8 @@ public static class ScanAnalysisKeys public const string ReachabilityUnionGraph = "analysis.reachability.union.graph"; public const string ReachabilityUnionCas = "analysis.reachability.union.cas"; + + public const string FileEntries = "analysis.files.entries"; + public const string EntropyReport = "analysis.entropy.report"; + public const string EntropyLayerSummary = "analysis.entropy.layer.summary"; } diff --git a/src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs b/src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs new file mode 100644 index 000000000..789e45ae6 --- /dev/null +++ b/src/Zastava/StellaOps.Zastava.Observer/Backend/RuntimeFactsClient.cs @@ -0,0 +1,71 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Zastava.Observer.Configuration; +using StellaOps.Zastava.Observer.Runtime; + +namespace StellaOps.Zastava.Observer.Backend; + +internal interface IRuntimeFactsClient +{ + Task PublishAsync(RuntimeFactsPublishRequest request, CancellationToken cancellationToken); +} + +internal sealed class RuntimeFactsClient : IRuntimeFactsClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + private readonly HttpClient httpClient; + private readonly IOptionsMonitor observerOptions; + private readonly ILogger logger; + + public RuntimeFactsClient( + HttpClient httpClient, + IOptionsMonitor observerOptions, + ILogger logger) + { + this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PublishAsync(RuntimeFactsPublishRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var reachability = observerOptions.CurrentValue.Reachability ?? new ReachabilityRuntimeOptions(); + var endpoint = reachability.Endpoint; + if (string.IsNullOrWhiteSpace(endpoint)) + { + throw new RuntimeFactsException("Reachability endpoint is not configured."); + } + + using var message = new HttpRequestMessage(HttpMethod.Post, endpoint); + + if (!string.IsNullOrWhiteSpace(reachability.AnalysisId)) + { + message.Headers.TryAddWithoutValidation("X-Analysis-Id", reachability.AnalysisId); + } + + var json = JsonSerializer.Serialize(request, SerializerOptions); + message.Content = new StringContent(json, Encoding.UTF8, "application/json"); + message.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using var response = await httpClient.SendAsync(message, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + return; + } + + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + logger.LogWarning("Runtime facts publish failed with status {Status}: {Body}", (int)response.StatusCode, body); + throw new RuntimeFactsException($"Runtime facts publish failed with status {(int)response.StatusCode}."); + } +} + diff --git a/src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs new file mode 100644 index 000000000..1fd5ffb6d --- /dev/null +++ b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ReachabilityRuntimeOptions.cs @@ -0,0 +1,55 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Zastava.Observer.Configuration; + +/// +/// Configuration for emitting runtime reachability facts to Signals. +/// +public sealed class ReachabilityRuntimeOptions +{ + /// + /// Enables runtime reachability fact publishing when true. + /// + public bool Enabled { get; set; } + + /// + /// Signals endpoint that accepts runtime facts (JSON or NDJSON). + /// + [Required(AllowEmptyStrings = false)] + public string Endpoint { get; set; } = "https://signals.internal/signals/runtime-facts"; + + /// + /// Required callgraph identifier used to correlate runtime facts with static graphs. + /// + [Required(AllowEmptyStrings = false)] + public string CallgraphId { get; set; } = string.Empty; + + /// + /// Optional analysis identifier forwarded as X-Analysis-Id. + /// + public string? AnalysisId { get; set; } + + /// + /// Maximum number of facts sent in a single publish attempt. + /// + [Range(1, 5000)] + public int BatchSize { get; set; } = 1000; + + /// + /// Maximum delay (seconds) before flushing a partially filled batch. + /// + [Range(typeof(double), "0.1", "30")] + public double FlushIntervalSeconds { get; set; } = 2; + + /// + /// Optional subject fallback when image data is missing. + /// + public string? SubjectScanId { get; set; } + + public string? SubjectImageDigest { get; set; } + + public string? SubjectComponent { get; set; } + + public string? SubjectVersion { get; set; } +} + diff --git a/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs index 5d7f6af2c..839a24795 100644 --- a/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs +++ b/src/Zastava/StellaOps.Zastava.Observer/Configuration/ZastavaObserverOptions.cs @@ -7,8 +7,8 @@ namespace StellaOps.Zastava.Observer.Configuration; /// /// Observer-specific configuration applied on top of the shared runtime options. /// -public sealed class ZastavaObserverOptions -{ +public sealed class ZastavaObserverOptions +{ public const string SectionName = "zastava:observer"; private const string DefaultContainerdSocket = "unix:///run/containerd/containerd.sock"; @@ -126,6 +126,12 @@ public sealed class ZastavaObserverOptions /// [Range(1, 128)] public int MaxEntrypointArguments { get; set; } = 32; + + /// + /// Runtime reachability fact publishing configuration. + /// + [Required] + public ReachabilityRuntimeOptions Reachability { get; init; } = new(); } public sealed class ZastavaObserverBackendOptions diff --git a/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs b/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs index e57c577ca..f4baea0fa 100644 --- a/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs +++ b/src/Zastava/StellaOps.Zastava.Observer/DependencyInjection/ObserverServiceCollectionExtensions.cs @@ -87,6 +87,14 @@ public static class ObserverServiceCollectionExtensions client.Timeout = TimeSpan.FromSeconds(Math.Clamp(backend.RequestTimeoutSeconds, 1, 120)); }); + services.AddHttpClient() + .ConfigureHttpClient((provider, client) => + { + var optionsMonitor = provider.GetRequiredService>(); + var observer = optionsMonitor.CurrentValue; + client.Timeout = TimeSpan.FromSeconds(Math.Clamp(observer.Backend.RequestTimeoutSeconds, 1, 120)); + }); + services.TryAddEnumerable(ServiceDescriptor.Singleton, ObserverRuntimeOptionsPostConfigure>()); // Surface environment + cache/manifest/secrets wiring diff --git a/src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs b/src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs new file mode 100644 index 000000000..7457611cd --- /dev/null +++ b/src/Zastava/StellaOps.Zastava.Observer/Runtime/RuntimeFactsBuilder.cs @@ -0,0 +1,357 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using System.Linq; +using System.Collections.Generic; +using StellaOps.Zastava.Core.Contracts; +using StellaOps.Zastava.Observer.Configuration; + +namespace StellaOps.Zastava.Observer.Runtime; + +internal static class RuntimeFactsBuilder +{ + public static RuntimeFactsPublishRequest? Build( + IReadOnlyCollection envelopes, + ReachabilityRuntimeOptions options) + { + ArgumentNullException.ThrowIfNull(envelopes); + ArgumentNullException.ThrowIfNull(options); + + if (!options.Enabled || envelopes.Count == 0) + { + return null; + } + + if (string.IsNullOrWhiteSpace(options.CallgraphId)) + { + return null; + } + + var facts = new List(envelopes.Count); + ReachabilitySubjectPayload? subject = null; + + foreach (var envelope in envelopes) + { + var fact = TryBuildFact(envelope); + if (fact is null) + { + continue; + } + + facts.Add(fact); + subject ??= BuildSubject(envelope, options); + } + + if (facts.Count == 0) + { + return null; + } + + subject ??= BuildFallbackSubject(options); + if (subject is null) + { + return null; + } + + facts.Sort(CompareFacts); + + return new RuntimeFactsPublishRequest + { + CallgraphId = options.CallgraphId.Trim(), + Subject = subject, + Events = facts + }; + } + + private static RuntimeFactEventPayload? TryBuildFact(RuntimeEventEnvelope envelope) + { + ArgumentNullException.ThrowIfNull(envelope); + + var evt = envelope.Event; + if (evt.Kind != RuntimeEventKind.ContainerStart || evt.Process is null) + { + return null; + } + + var symbolId = BuildSymbolId(evt.Process); + if (string.IsNullOrWhiteSpace(symbolId)) + { + return null; + } + + var fact = new RuntimeFactEventPayload + { + SymbolId = symbolId, + CodeId = Normalize(evt.Process.BuildId), + BuildId = Normalize(evt.Process.BuildId), + LoaderBase = null, + Purl = null, + SymbolDigest = null, + HitCount = 1, + ObservedAt = evt.When, + ProcessId = evt.Process.Pid, + ProcessName = ResolveProcessName(evt.Process.Entrypoint), + ContainerId = evt.Workload.ContainerId, + Metadata = BuildMetadata(evt.Process) + }; + + return fact; + } + + private static string? BuildSymbolId(RuntimeProcess process) + { + if (!string.IsNullOrWhiteSpace(process.BuildId)) + { + return $"sym:binary:{process.BuildId.Trim().ToLowerInvariant()}"; + } + + var trace = process.EntryTrace?.FirstOrDefault(t => + !string.IsNullOrWhiteSpace(t.Target) || !string.IsNullOrWhiteSpace(t.File)); + + var seed = trace?.Target ?? trace?.File ?? process.Entrypoint.FirstOrDefault(); + if (string.IsNullOrWhiteSpace(seed)) + { + return null; + } + + var stable = ComputeStableFragment(seed); + return $"sym:shell:{stable}"; + } + + private static ReachabilitySubjectPayload? BuildSubject(RuntimeEventEnvelope envelope, ReachabilityRuntimeOptions options) + { + var workload = envelope.Event.Workload; + var imageRef = workload.ImageRef; + + var digest = ExtractImageDigest(imageRef); + var (component, version) = ExtractComponentAndVersion(imageRef); + + digest ??= Normalize(options.SubjectImageDigest); + component ??= Normalize(options.SubjectComponent); + version ??= Normalize(options.SubjectVersion); + + var scanId = Normalize(options.SubjectScanId); + + if (string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(scanId) + && (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version))) + { + return null; + } + + return new ReachabilitySubjectPayload + { + ScanId = scanId, + ImageDigest = digest, + Component = component, + Version = version + }; + } + + private static ReachabilitySubjectPayload? BuildFallbackSubject(ReachabilityRuntimeOptions options) + { + var digest = Normalize(options.SubjectImageDigest); + var component = Normalize(options.SubjectComponent); + var version = Normalize(options.SubjectVersion); + var scanId = Normalize(options.SubjectScanId); + + if (string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(scanId) + && (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version))) + { + return null; + } + + return new ReachabilitySubjectPayload + { + ScanId = scanId, + ImageDigest = digest, + Component = component, + Version = version + }; + } + + private static Dictionary? BuildMetadata(RuntimeProcess process) + { + if (process.EntryTrace is null || process.EntryTrace.Count == 0) + { + return null; + } + + var sb = new StringBuilder(); + foreach (var trace in process.EntryTrace) + { + if (trace is null) + { + continue; + } + + var op = string.IsNullOrWhiteSpace(trace.Op) ? "exec" : trace.Op; + var target = trace.Target ?? trace.File; + if (string.IsNullOrWhiteSpace(target)) + { + continue; + } + + if (sb.Length > 0) + { + sb.Append(" | "); + } + sb.Append(op).Append(':').Append(target); + } + + if (sb.Length == 0) + { + return null; + } + + return new Dictionary(StringComparer.Ordinal) + { + ["entryTrace"] = sb.ToString() + }; + } + + private static string? ExtractImageDigest(string? imageRef) + { + if (string.IsNullOrWhiteSpace(imageRef)) + { + return null; + } + + var digestStart = imageRef.IndexOf("sha256:", StringComparison.OrdinalIgnoreCase); + if (digestStart >= 0) + { + return imageRef[digestStart..].Trim(); + } + + var atIndex = imageRef.IndexOf('@'); + if (atIndex > 0 && atIndex + 1 < imageRef.Length) + { + return imageRef[(atIndex + 1)..].Trim(); + } + + return null; + } + + private static (string? Component, string? Version) ExtractComponentAndVersion(string? imageRef) + { + if (string.IsNullOrWhiteSpace(imageRef)) + { + return (null, null); + } + + var lastSlash = imageRef.LastIndexOf('/'); + var lastColon = imageRef.LastIndexOf(':'); + + if (lastColon < 0 || lastColon < lastSlash) + { + return (null, null); + } + + var component = imageRef[(lastSlash + 1)..lastColon]; + var version = imageRef[(lastColon + 1)..]; + + if (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version)) + { + return (null, null); + } + + return (component.Trim(), version.Trim()); + } + + private static string? ResolveProcessName(IReadOnlyList? entrypoint) + { + if (entrypoint is null || entrypoint.Count == 0) + { + return null; + } + + var first = entrypoint[0]; + if (string.IsNullOrWhiteSpace(first)) + { + return null; + } + + var lastSlash = first.LastIndexOf('/'); + return lastSlash >= 0 ? first[(lastSlash + 1)..] : first; + } + + private static string ComputeStableFragment(string seed) + { + var normalized = seed.Trim().ToLowerInvariant(); + var bytes = Encoding.UTF8.GetBytes(normalized); + Span hash = stackalloc byte[32]; + _ = SHA256.TryHashData(bytes, hash, out _); + return Convert.ToHexString(hash[..16]).ToLowerInvariant(); + } + + private static int CompareFacts(RuntimeFactEventPayload left, RuntimeFactEventPayload right) + { + var timeComparison = Nullable.Compare(left.ObservedAt, right.ObservedAt); + if (timeComparison != 0) + { + return timeComparison; + } + + return string.Compare(left.SymbolId, right.SymbolId, StringComparison.Ordinal); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); +} + +internal sealed class RuntimeFactsPublishRequest +{ + public ReachabilitySubjectPayload Subject { get; set; } = new(); + + public string CallgraphId { get; set; } = string.Empty; + + public List Events { get; set; } = new(); +} + +internal sealed class ReachabilitySubjectPayload +{ + public string? ScanId { get; set; } + + public string? ImageDigest { get; set; } + + public string? Component { get; set; } + + public string? Version { get; set; } +} + +internal sealed class RuntimeFactEventPayload +{ + public string SymbolId { get; set; } = string.Empty; + + public string? CodeId { get; set; } + + public string? SymbolDigest { get; set; } + + public string? Purl { get; set; } + + public string? BuildId { get; set; } + + public string? LoaderBase { get; set; } + + public int? ProcessId { get; set; } + + public string? ProcessName { get; set; } + + public string? SocketAddress { get; set; } + + public string? ContainerId { get; set; } + + public string? EvidenceUri { get; set; } + + public int HitCount { get; set; } = 1; + + public DateTimeOffset? ObservedAt { get; set; } + + public Dictionary? Metadata { get; set; } +} + +internal sealed class RuntimeFactsException : Exception +{ + public RuntimeFactsException(string message, Exception? innerException = null) : base(message, innerException) + { + } +} diff --git a/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs b/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs index e681d3ca7..b8805526b 100644 --- a/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs +++ b/src/Zastava/StellaOps.Zastava.Observer/Worker/RuntimeEventDispatchService.cs @@ -13,6 +13,7 @@ internal sealed class RuntimeEventDispatchService : BackgroundService { private readonly IRuntimeEventBuffer buffer; private readonly IRuntimeEventsClient eventsClient; + private readonly IRuntimeFactsClient runtimeFactsClient; private readonly IOptionsMonitor observerOptions; private readonly TimeProvider timeProvider; private readonly ILogger logger; @@ -20,12 +21,14 @@ internal sealed class RuntimeEventDispatchService : BackgroundService public RuntimeEventDispatchService( IRuntimeEventBuffer buffer, IRuntimeEventsClient eventsClient, + IRuntimeFactsClient runtimeFactsClient, IOptionsMonitor observerOptions, TimeProvider timeProvider, ILogger logger) { this.buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); this.eventsClient = eventsClient ?? throw new ArgumentNullException(nameof(eventsClient)); + this.runtimeFactsClient = runtimeFactsClient ?? throw new ArgumentNullException(nameof(runtimeFactsClient)); this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions)); this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -116,14 +119,17 @@ internal sealed class RuntimeEventDispatchService : BackgroundService return; } - var request = new RuntimeEventsIngestRequest - { - BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}", - Events = batch.Select(item => item.Envelope).ToArray() - }; - try { + var envelopes = batch.Select(item => item.Envelope).ToArray(); + var factsPublished = await TryPublishRuntimeFactsAsync(envelopes, cancellationToken).ConfigureAwait(false); + + var request = new RuntimeEventsIngestRequest + { + BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}", + Events = envelopes + }; + var result = await eventsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false); if (result.Success) { @@ -132,10 +138,11 @@ internal sealed class RuntimeEventDispatchService : BackgroundService await item.CompleteAsync().ConfigureAwait(false); } - logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}).", + logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}, runtimeFacts={FactsPublished}).", request.BatchId, result.Accepted, - result.Duplicates); + result.Duplicates, + factsPublished); } else if (result.RateLimited) { @@ -166,6 +173,38 @@ internal sealed class RuntimeEventDispatchService : BackgroundService } } + private async Task TryPublishRuntimeFactsAsync(RuntimeEventEnvelope[] envelopes, CancellationToken cancellationToken) + { + if (envelopes.Length == 0) + { + return false; + } + + var options = observerOptions.CurrentValue.Reachability; + var request = RuntimeFactsBuilder.Build(envelopes, options); + if (request is null) + { + return false; + } + + try + { + await runtimeFactsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false); + logger.LogDebug("Published {Count} runtime facts (callgraphId={CallgraphId}).", request.Events.Count, request.CallgraphId); + return true; + } + catch (RuntimeFactsException ex) when (!cancellationToken.IsCancellationRequested) + { + logger.LogWarning(ex, "Runtime facts publish failed; batch will be retried."); + throw; + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + logger.LogWarning(ex, "Runtime facts publish encountered an unexpected error; batch will be retried."); + throw; + } + } + private async Task RequeueBatchAsync(IEnumerable batch, CancellationToken cancellationToken) { foreach (var item in batch) diff --git a/src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs b/src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs new file mode 100644 index 000000000..f32f81896 --- /dev/null +++ b/src/Zastava/__Tests/StellaOps.Zastava.Observer.Tests/Runtime/RuntimeFactsBuilderTests.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using StellaOps.Zastava.Core.Contracts; +using StellaOps.Zastava.Observer.Configuration; +using StellaOps.Zastava.Observer.Runtime; +using Xunit; + +namespace StellaOps.Zastava.Observer.Tests.Runtime; + +public sealed class RuntimeFactsBuilderTests +{ + [Fact] + public void Build_UsesBuildIdAndDigest() + { + var options = new ReachabilityRuntimeOptions + { + Enabled = true, + CallgraphId = "cg-001" + }; + + var runtimeEvent = CreateRuntimeEvent( + imageRef: "ghcr.io/example/api@sha256:deadbeef", + buildId: "beadfeed", + when: DateTimeOffset.Parse("2025-11-26T12:00:00Z")); + + var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + var result = RuntimeFactsBuilder.Build(new[] { envelope }, options); + + Assert.NotNull(result); + Assert.Equal("cg-001", result!.CallgraphId); + Assert.Equal("sha256:deadbeef", result.Subject.ImageDigest); + Assert.Single(result.Events); + Assert.StartsWith("sym:binary:beadfeed", result.Events[0].SymbolId, StringComparison.Ordinal); + } + + [Fact] + public void Build_ParsesComponentAndVersion_WhenTagPresent() + { + var options = new ReachabilityRuntimeOptions + { + Enabled = true, + CallgraphId = "cg-002" + }; + + var runtimeEvent = CreateRuntimeEvent( + imageRef: "registry.local/team/web:1.2.3", + buildId: null, + when: DateTimeOffset.Parse("2025-11-26T12:01:00Z")); + + var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + var result = RuntimeFactsBuilder.Build(new[] { envelope }, options); + + Assert.NotNull(result); + Assert.Equal("web", result!.Subject.Component); + Assert.Equal("1.2.3", result.Subject.Version); + Assert.Single(result.Events); + Assert.StartsWith("sym:shell:", result.Events[0].SymbolId, StringComparison.Ordinal); + } + + private static RuntimeEvent CreateRuntimeEvent(string imageRef, string? buildId, DateTimeOffset when) + { + var process = new RuntimeProcess + { + Pid = 1234, + Entrypoint = new[] { "/entrypoint.sh", "-c", "echo hi" }, + EntryTrace = new[] { new RuntimeEntryTrace { File = "/entrypoint.sh", Op = "exec", Target = "/entrypoint.sh" } }, + BuildId = buildId + }; + + return new RuntimeEvent + { + EventId = "evt-1", + When = when, + Kind = RuntimeEventKind.ContainerStart, + Tenant = "tenant-a", + Node = "node-a", + Runtime = new RuntimeEngine { Engine = "containerd", Version = "1.7" }, + Workload = new RuntimeWorkload + { + Platform = "kubernetes", + Namespace = "default", + Pod = "pod-a", + Container = "api", + ContainerId = "containerd://abc", + ImageRef = imageRef + }, + Process = process, + LoadedLibraries = Array.Empty(), + Evidence = Array.Empty() + }; + } +} +