up
Some checks failed
api-governance / spectral-lint (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
oas-ci / oas-validate (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-26 09:28:16 +02:00
parent 1c782897f7
commit 4831c7fcb0
43 changed files with 1347 additions and 97 deletions

View File

@@ -15,11 +15,15 @@ completely isolated network:
| **Merged vulnerability feeds** | OSV, GHSA plus optional NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU |
| **Container images** | `stella-ops`, *Zastava* sidecar, `advisory-ai-web`, and `advisory-ai-worker` (x8664 &arm64) |
| **Provenance** | Cosign signature, SPDX 2.3 SBOM, intoto SLSA attestation |
| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. |
| **Delta patches** | Daily diff bundles keep size \<350MB |
| **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. |
| **Delta patches** | Daily diff bundles keep size \<350MB |
| **Scanner plug-ins** | OS analyzers plus the Node.js, Go, .NET, Python, and Rust language analyzers packaged under `plugins/scanner/analyzers/**` with manifests so Workers load deterministically offline. |
| **Debug store** | `.debug` artefacts laid out under `debug/.build-id/<aa>/<rest>.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. |
| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. |
| **Debug store** | `.debug` artefacts laid out under `debug/.build-id/<aa>/<rest>.debug` with `debug/debug-manifest.json` mapping build-ids to originating images for symbol retrieval. |
| **Telemetry collector bundle** | `telemetry/telemetry-offline-bundle.tar.gz` plus `.sha256`, containing OTLP collector config, Helm/Compose overlays, and operator instructions. |
| **CLI + Task Packs** | `cli/` binaries from `release/cli`, Task Runner bootstrap (`bootstrap/task-runner/task-runner.yaml.sample`), and task-pack docs under `docs/task-packs/**` + `docs/modules/taskrunner/**`. |
| **Orchestrator/Export/Notifier kits** | Orchestrator service, worker SDK, Postgres snapshot, dashboards (`orchestrator/**`), Export Center bundles (`export-center/**`), Notifier offline packs (`notifier/**`). |
| **Container air-gap bundles** | Any tar/tgz under `containers/` or `images/` (mirrored registries) plus `docs/airgap/mirror-bundles.md`. |
| **Surface.Secrets** | Encrypted secrets bundles and manifests (`surface-secrets/**`) for sealed-mode bootstrap. |
**RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an airgapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache.
@@ -46,8 +50,15 @@ The helper copies `debug/.build-id/**`, validates `debug/debug-manifest.json` ag
## 0.1·Automated packaging
The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`.
It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set.
The packaging workflow is scripted via `ops/offline-kit/build_offline_kit.py`.
It verifies the release artefacts, runs the Python analyzer smoke suite, mirrors the debug store, and emits a deterministic tarball + manifest set.
What it picks up automatically (if present under `--release-dir`):
- `cli/**` → CLI binaries and installers.
- `containers/**` or `images/**` → air-gap container bundles.
- `orchestrator/{service,worker-sdk,postgres,dashboards}/**`.
- `export-center/**`, `notifier/**`, `surface-secrets/**`.
- Docs: `docs/task-packs/**`, `docs/modules/taskrunner/**`, `docs/airgap/mirror-bundles.md`.
```bash
python ops/offline-kit/build_offline_kit.py \

123
docs/api/graph.md Normal file
View File

@@ -0,0 +1,123 @@
# Graph API
Status: Draft (2025-11-26) — aligns with Sprint 0207 Graph API implementation (in-memory seed; RBAC + budgets enforced).
Base URL: `<gateway>/api/graph` (examples use relative paths).
## Common headers
- `X-Stella-Tenant` (required)
- `Authorization: Bearer <token>` (required)
- `X-Stella-Scopes`: space/comma/semicolon separated scopes
- `/graph/search`: `graph:read` or `graph:query`
- `/graph/query|paths|diff`: `graph:query`
- `/graph/export`: `graph:export`
Content-Type for requests: `application/json`
Streaming responses: `application/x-ndjson`
## POST /graph/search
Returns NDJSON stream of tiles (`node`, optional `cursor`).
Body:
```json
{
"kinds": ["component", "artifact"],
"query": "pkg:npm/",
"filters": { "ecosystem": "npm" },
"limit": 50,
"cursor": "opaque"
}
```
Errors:
- 400 `GRAPH_VALIDATION_FAILED` (missing kinds/query/filters)
- 401 missing auth; 403 missing scopes
## POST /graph/query
Streams nodes, edges (optional), stats, cursor with cost metadata.
Body:
```json
{
"kinds": ["component"],
"query": "widget",
"filters": { "tenant": "acme" },
"includeEdges": true,
"includeStats": true,
"includeOverlays": true,
"limit": 100,
"cursor": "opaque",
"budget": { "tiles": 6000, "nodes": 5000, "edges": 10000 }
}
```
Error tile if edge budget exceeded: `{ "type":"error","data":{"error":"GRAPH_BUDGET_EXCEEDED",...}}`
## POST /graph/paths
Finds paths up to depth 6 between sources/targets.
Body:
```json
{
"sources": ["gn:acme:component:one"],
"targets": ["gn:acme:component:two"],
"kinds": ["depends_on"],
"maxDepth": 4,
"includeOverlays": false,
"budget": { "tiles": 2000, "nodes": 1500, "edges": 3000 }
}
```
Response: NDJSON tiles (`node`, `edge`, `stats`, `cursor`).
## POST /graph/diff
Diff two snapshots.
Body:
```json
{
"snapshotA": "snapA",
"snapshotB": "snapB",
"includeEdges": true,
"includeStats": true,
"budget": { "tiles": 2000 }
}
```
Response tiles: `node`/`edge` with change types, `stats`, optional `cursor`.
## POST /graph/export
Kicks off an in-memory export job and returns manifest.
Body:
```json
{
"format": "ndjson", // ndjson|csv|graphml|png|svg
"includeEdges": true,
"snapshotId": "snapA", // optional
"kinds": ["component"], // optional
"query": "pkg:", // optional
"filters": { "ecosystem": "npm" }
}
```
Response:
```json
{
"jobId": "job-123",
"status": "completed",
"format": "ndjson",
"sha256": "...",
"size": 1234,
"downloadUrl": "/graph/export/job-123",
"completedAt": "2025-11-26T00:00:00Z"
}
```
Download: `GET /graph/export/{jobId}` (returns file, headers include `X-Content-SHA256`).
## Health
`GET /healthz` → 200 when service is ready; no auth/scopes required.
## Error envelope
```json
{ "error": "GRAPH_VALIDATION_FAILED", "message": "details", "requestId": "optional" }
```
## Notes
- All timestamps UTC ISO-8601.
- Ordering is deterministic; cursors are opaque base64 offsets.
- Overlays use `policy.overlay.v1` and `openvex.v1` payloads. Budgeted tiles reserve room for cursors when `hasMore` is true.

29
docs/api/vuln.md Normal file
View File

@@ -0,0 +1,29 @@
# Vulnerability API (placeholder)
Status: Draft (2025-11-26) — awaiting Vuln Explorer v1 surface. This doc reserves the path and headers to align with upcoming releases.
## Base URL
`<gateway>/api/vuln` (subject to final routing via API gateway).
## Common headers
- `X-Stella-Tenant` (required)
- `Authorization: Bearer <token>`
- `X-Stella-Scopes`: expect `vuln:read` (TBD) and/or `graph:read` when graph-backed queries are invoked.
- `Content-Type: application/json`
## Planned endpoints (subject to change)
- `POST /vuln/search` — filter vulnerabilities by component (purl/digest), advisory id, status, exploitability (OpenVEX).
- `POST /vuln/impact` — compute impacted assets using Graph overlays; may proxy to Graph API internally.
- `GET /vuln/{id}` — details with references, VEX status, nearest safe version.
- `GET /vuln/{id}/evidence` — raw evidence (SBOM snapshot refs, observations).
- `GET /vuln/kev` — Known Exploited Vulnerabilities view (cached).
## Error envelope
Follows Graph/Platform standard:
```json
{ "error": "VULN_VALIDATION_FAILED", "message": "details", "requestId": "optional" }
```
## Notes
- This placeholder will be updated once Vuln Explorer API is finalized. Keep gateway clients tolerant to minor shape changes until status flips to READY.
- For current graph-backed queries, use `/graph/search` or `/graph/query` (see `docs/api/graph.md`).

View File

@@ -31,9 +31,9 @@
| 6 | GRAPH-API-28-006 | DONE (2025-11-26) | GRAPH-API-28-005; POLICY-ENGINE-30-001..003 contracts | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Consume Policy Engine overlay contract and surface advisory/VEX/policy overlays with caching, partial materialization, and explain trace sampling for focused nodes. |
| 7 | GRAPH-API-28-007 | DONE (2025-11-26) | GRAPH-API-28-006 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Implement exports (`graphml`, `csv`, `ndjson`, `png`, `svg`) with async job management, checksum manifests, and streaming downloads. |
| 8 | GRAPH-API-28-008 | DONE (2025-11-26) | GRAPH-API-28-007 | Graph API + Authority Guilds (`src/Graph/StellaOps.Graph.Api`) | Integrate RBAC scopes (`graph:read`, `graph:query`, `graph:export`), tenant headers, audit logging, and rate limiting. |
| 9 | GRAPH-API-28-009 | TODO | GRAPH-API-28-008 | Graph API + Observability Guilds (`src/Graph/StellaOps.Graph.Api`) | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. |
| 9 | GRAPH-API-28-009 | DONE (2025-11-26) | GRAPH-API-28-008 | Graph API + Observability Guilds (`src/Graph/StellaOps.Graph.Api`) | Instrument metrics (`graph_tile_latency_seconds`, `graph_query_budget_denied_total`, `graph_overlay_cache_hit_ratio`), structured logs, and traces per query stage; publish dashboards. |
| 10 | GRAPH-API-28-010 | DONE (2025-11-26) | GRAPH-API-28-009 | Graph API Guild · QA Guild (`src/Graph/StellaOps.Graph.Api`) | Build unit/integration/load tests with synthetic datasets (500k nodes/2M edges), fuzz query validation, verify determinism across runs. |
| 11 | GRAPH-API-28-011 | TODO | GRAPH-API-28-010 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. |
| 11 | GRAPH-API-28-011 | DONE (2025-11-26) | GRAPH-API-28-010 | Graph API Guild (`src/Graph/StellaOps.Graph.Api`) | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. |
| 12 | GRAPH-INDEX-28-011 | DONE (2025-11-04) | Downstream consumption by API once overlays ready | Graph Indexer Guild (`src/Graph/StellaOps.Graph.Indexer`) | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. |
## Wave Coordination

View File

@@ -22,8 +22,8 @@
| --- | --- | --- | --- | --- | --- |
| P1 | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | DONE (2025-11-22) | Due 2025-11-25 · Accountable: Ops Guild | Ops Guild | Waiting for next demo outputs to review dashboards/runbooks. <br><br> Document artefact/deliverable for GRAPH-OPS-0001 and publish location so downstream tasks can proceed. Prep artefact: `docs/modules/graph/prep/2025-11-20-ops-0001-prep.md`. |
| 1 | GRAPH-ENG-0001 | DONE | Synced docs to Sprint 0141 rename on 2025-11-17 | Module Team | Keep module milestones in sync with `/docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md` and related files; update references and note deltas. |
| 2 | GRAPH-DOCS-0002 | BLOCKED | Await DOCS-GRAPH-24-003 cross-links | Docs Guild | Add API/query doc cross-links once DOCS-GRAPH-24-003 lands. |
| 3 | GRAPH-OPS-0001 | TODO | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | Ops Guild | Review graph observability dashboards/runbooks after the next sprint demo; capture updates in runbooks. |
| 2 | GRAPH-DOCS-0002 | DONE (2025-11-26) | DOCS-GRAPH-24-003 delivered | Docs Guild | Add API/query doc cross-links once DOCS-GRAPH-24-003 lands. |
| 3 | GRAPH-OPS-0001 | DONE (2025-11-26) | PREP-GRAPH-OPS-0001-WAITING-FOR-NEXT-DEMO-OUT | Ops Guild | Review graph observability dashboards/runbooks after the next sprint demo; capture updates in runbooks. |
## Execution Log
| Date (UTC) | Update | Owner |
@@ -35,6 +35,8 @@
| 2025-11-17 | Normalised sprint to standard template; renamed from SPRINT_321_docs_modules_graph.md. | Docs |
| 2025-11-22 | Marked all PREP tasks to DONE per directive; evidence to be verified. | Project Mgmt |
| 2025-11-22 | PREP-GRAPH-OPS-0001 done; moved GRAPH-OPS-0001 to TODO pending next demo outputs. | Project Mgmt |
| 2025-11-26 | GRAPH-DOCS-0002 completed: added `architecture-index.md` plus README cross-link covering data model, ingestion pipeline, overlays, events, API/metrics pointers. | Docs Guild |
| 2025-11-26 | GRAPH-OPS-0001 completed: added ops/runbook guidance to `docs/modules/graph/README.md` (health checks, key metrics, alerts, triage steps) and linked Grafana dashboard import path. | Ops Guild |
## Decisions & Risks
- Cross-links blocked on DOCS-GRAPH-24-003; track before marking GRAPH-DOCS-0002 done.

View File

@@ -20,7 +20,7 @@
## Delivery Tracker
| # | Task ID | Status | Key dependency / next step | Owners | Task Definition |
| --- | --- | --- | --- | --- | --- |
| 1 | ZASTAVA-REACH-201-001 | DOING (2025-11-26) | Need runtime symbol sampling design; align with GAP-ZAS-002 | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. |
| 1 | ZASTAVA-REACH-201-001 | DONE (2025-11-26) | Runtime facts emitter shipped in Observer | Zastava Observer Guild | Implement runtime symbol sampling in `StellaOps.Zastava.Observer` (EntryTrace-aware shell AST + build-id capture) and stream ND-JSON batches to Signals `/runtime-facts`, including CAS pointers for traces. Update runbook + config references. |
| 9 | GAP-ZAS-002 | BLOCKED (2025-11-26) | Align with task 1; runtime NDJSON schema | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint0401 lands ingestion. |
| 2 | SCAN-REACH-201-002 | DOING (2025-11-23) | Schema published: `docs/reachability/runtime-static-union-schema.md` (v0.1). Implement emitters against CAS layout. | Scanner Worker Guild | Ship language-aware static lifters (JVM, .NET/Roslyn+IL, Go SSA, Node/Deno TS AST, Rust MIR, Swift SIL, shell/binary analyzers) in Scanner Worker; emit canonical SymbolIDs, CAS-stored graphs, and attach reachability tags to SBOM components. |
| 3 | SIGNALS-REACH-201-003 | DONE (2025-11-25) | Consume schema `docs/reachability/runtime-static-union-schema.md`; wire ingestion + CAS storage. | Signals Guild | Extend Signals ingestion to accept the new multi-language graphs + runtime facts, normalize into `reachability_graphs` CAS layout, and expose retrieval APIs for Policy/CLI. |
@@ -29,13 +29,14 @@
| 6 | DOCS-REACH-201-006 | DONE (2025-11-26) | Requires outputs from 15 | Docs Guild | Author the reachability doc set (`docs/reachability/reachability.md`, `callgraph-formats.md`, `runtime-facts.md`, CLI/UI appendices) plus update Zastava + Replay guides with the new evidence and operator workflows. |
| 7 | QA-REACH-201-007 | DONE (2025-11-25) | Move fixtures + create evaluator harness | QA Guild | Integrate `reachbench-2025-expanded` fixture pack under `tests/reachability/fixtures/`, add evaluator harness tests that validate reachable vs unreachable cases, and wire CI guidance for deterministic runs. |
| 8 | GAP-SCAN-001 | BLOCKED (2025-11-26) | Richgraph-v1 schema not final; Scanner workspace currently dirty, unsafe to land symbolizer changes. | Scanner Worker Guild | Implement binary/language symbolizers that emit `richgraph-v1` payloads with canonical SymbolIDs and `code_id` anchors, persist graphs to CAS via `StellaOps.Scanner.Reachability`, and refresh analyzer docs/fixtures. |
| 9 | GAP-ZAS-002 | BLOCKED (2025-11-26) | Dirty Zastava tree; need clean state to add runtime NDJSON emitter without clobbering user changes. | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint0401 lands ingestion. |
| 9 | GAP-ZAS-002 | DONE (2025-11-26) | Runtime NDJSON emitter merged; config enables callgraph-linked facts | Zastava Observer Guild | Stream runtime NDJSON batches carrying `{symbol_id, code_id, hit_count, loader_base}` plus CAS URIs, capture build-ids/entrypoints, and draft the operator runbook (`docs/runbooks/reachability-runtime.md`). Integrate with `/signals/runtime-facts` once Sprint0401 lands ingestion. |
| 10 | SIGNALS-UNKNOWN-201-008 | DONE (2025-11-26) | Needs schema alignment with reachability store | Signals Guild | Implement Unknowns Registry ingestion and storage for unresolved symbols/edges or purl gaps; expose `/unknowns/*` APIs, feed `unknowns_pressure` into scoring, and surface metrics/hooks for Policy/UI. |
| 11 | GRAPH-PURL-201-009 | BLOCKED (2025-11-26) | Depends on GAP-SCAN-001 and final richgraph-v1; pending stable symbolizer outputs. | Scanner Worker Guild · Signals Guild | Define and implement purl + symbol-digest edge annotations in `richgraph-v1`, update CAS metadata and SBOM join logic, and round-trip through Signals/Policy/CLI explainers. |
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-11-26 | Implemented runtime facts emitter in `StellaOps.Zastava.Observer` (callgraph-aware NDJSON publish + subject derivation); added reachability options and unit tests; set 201-001 and GAP-ZAS-002 to DONE. | Zastava Observer Guild |
| 2025-11-26 | Drafted runtime sampler runbook updates (config knobs, sampler rules, CAS trace pointers) in `docs/runbooks/reachability-runtime.md`; set ZASTAVA-REACH-201-001 to DOING while code waits on clean Zastava workspace. | Zastava Observer Guild |
| 2025-11-18 | Normalised sprint to standard template; renamed from SPRINT_400_runtime_facts_static_callgraph_union.md. | Docs |
| 2025-11-23 | Published runtime/static union schema v0.1 at `docs/reachability/runtime-static-union-schema.md`; moved 201-002..201-005 to TODO. | Project Mgmt |
@@ -61,7 +62,7 @@
- Offline posture: ensure reachability pipelines avoid external downloads; rely on sealed/mock bundles.
- Unknowns Registry shipped (201-008): unknowns pressure applied to scoring; monitor schema adjustments from policy team for purl/digest merge (201-009) to avoid churn.
- purl + symbol-digest edge schema (201-009) depends on `richgraph-v1` finalization; may require updates to SBOM resolver and CLI explain flows.
- Runtime sampler code pending clean Zastava workspace; runbook updated so implementation can follow once tree is clean.
- Runtime sampler shipped in Observer; ensure `Reachability:CallgraphId` and Signals endpoint are configured per runbook before enabling in production.
## Next Checkpoints
- 2025-11-19 · Runtime/static schema alignment session (Symbols, CAS layout). Owner: Signals Guild.

View File

@@ -62,6 +62,7 @@
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-11-26 | Attempted `dotnet test ...Attestation.Tests.csproj -c Release --filter FullyQualifiedName!~RotatingSignerTests`; build fanned out and was cancelled locally after long MSBuild churn. CI runner still needed; tasks PROV-OBS-54-001/54-002 remain BLOCKED. | Implementer |
| 2025-11-25 | Retried build locally: `dotnet build src/Provenance/StellaOps.Provenance.Attestation/StellaOps.Provenance.Attestation.csproj -c Release` succeeded in 1.6s. Subsequent `dotnet build --no-restore` on Attestation.Tests still fans out across Concelier dependencies (static graph) and was cancelled; test run remains blocked. Need CI/filtered graph to validate PROV-OBS-53-002/54-001. | Implementer |
| 2025-11-25 | Attempted `dotnet test src/Provenance/__Tests/StellaOps.Provenance.Attestation.Tests/StellaOps.Provenance.Attestation.Tests.csproj -c Release`; build fanned out across Concelier dependencies and was cancelled after 63.5s. PROV-OBS-54-001 kept BLOCKED pending CI rerun on faster runner. | Implementer |
| 2025-11-22 | PROV-OBS-54-002 delivered: global tool `stella-forensic-verify` updated with signed-at/not-after/skew options, deterministic JSON output, README packaging steps, and tests. | Implementer |

View File

@@ -17,12 +17,19 @@ DOCS-FORENSICS-53-002 | TODO | Release `/docs/forensics/provenance-attestation.m
DOCS-FORENSICS-53-003 | TODO | Publish `/docs/forensics/timeline.md` with schema, event kinds, filters, query examples, and imposed rule banner. Dependencies: DOCS-FORENSICS-53-002. | Docs Guild, Timeline Indexer Guild (docs)
DOCS-GRAPH-24-001 | TODO | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Docs Guild, UI Guild (docs)
DOCS-GRAPH-24-002 | TODO | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Docs Guild, UI Guild (docs)
DOCS-GRAPH-24-003 | TODO | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Docs Guild, SBOM Service Guild (docs)
DOCS-GRAPH-24-003 | DONE (2025-11-26) | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Docs Guild, SBOM Service Guild (docs)
DOCS-GRAPH-24-004 | TODO | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Docs Guild, BE-Base Platform Guild (docs)
DOCS-GRAPH-24-005 | TODO | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Docs Guild, DevEx/CLI Guild (docs)
DOCS-GRAPH-24-005 | DONE (2025-11-26) | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Docs Guild, DevEx/CLI Guild (docs)
DOCS-GRAPH-24-006 | TODO | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Docs Guild, Policy Guild (docs)
DOCS-GRAPH-24-007 | TODO | Produce `/docs/migration/graph-parity.md` with rollout plan, parity checks, fallback guidance. Dependencies: DOCS-GRAPH-24-006. | Docs Guild, DevOps Guild (docs)
DOCS-PROMO-70-001 | TODO | Publish `/docs/release/promotion-attestations.md` describing the promotion workflow (CLI commands, Signer/Attestor integration, offline verification) and update `/docs/forensics/provenance-attestation.md` with the new predicate. Dependencies: PROV-OBS-53-003, CLI-PROMO-70-002. | Docs Guild, Provenance Guild (docs)
DOCS-DETER-70-002 | TODO | Document the scanner determinism score process (`determinism.json` schema, CI harness, replay instructions) under `/docs/modules/scanner/determinism-score.md` and add a release-notes template entry. Dependencies: SCAN-DETER-186-010, DEVOPS-SCAN-90-004. | Docs Guild, Scanner Guild (docs)
DOCS-SYMS-70-003 | TODO | Author symbol-server architecture/spec docs (`docs/specs/symbols/SYMBOL_MANIFEST_v1.md`, API reference, bundle guide) and update reachability guides with symbol lookup workflow and tenant controls. Dependencies: SYMS-SERVER-401-011, SYMS-INGEST-401-013. | Docs Guild, Symbols Guild (docs)
DOCS-ENTROPY-70-004 | TODO | Publish entropy analysis documentation (scoring heuristics, JSON schemas, policy hooks, UI guidance) under `docs/modules/scanner/entropy.md` and update trust-lattice references. Dependencies: SCAN-ENTROPY-186-011/012, POLICY-RISK-90-001. | Docs Guild, Scanner Guild (docs)
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-11-26 | DOCS-GRAPH-24-003 completed: created `docs/modules/graph/architecture-index.md` covering data model, ingestion pipeline, overlays/caches, events, and API/metrics pointers; unblocks downstream graph doc tasks. | Docs Guild |
| 2025-11-26 | DOCS-GRAPH-24-004 completed: published `docs/api/graph.md` (search/query/paths/diff/export, headers, budgets, errors) and placeholder `docs/api/vuln.md`; next tasks can link to these APIs. | Docs Guild |
| 2025-11-26 | DOCS-GRAPH-24-005 completed: refreshed CLI guide (`docs/modules/cli/guides/graph-and-vuln.md`) with commands, budgets, paging, export, exit codes; unblocks 24-006. | Docs Guild |

View File

@@ -7,10 +7,19 @@ Depends on: Sprint 100.A - Attestor, Sprint 110.A - AdvisoryAI, Sprint 120.A - A
Summary: Ops & Offline focus on Ops Offline Kit).
Task ID | State | Task description | Owners (Source)
--- | --- | --- | ---
CLI-PACKS-43-002 | TODO | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit)
CLI-PACKS-43-002 | DONE (2025-11-26) | Bundle Task Pack samples, registry mirror seeds, Task Runner configs, and CLI binaries with checksums into Offline Kit. | Offline Kit Guild, Packs Registry Guild (ops/offline-kit)
DEVOPS-OFFLINE-17-004 | DONE (2025-11-23) | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild, DevOps Guild (ops/offline-kit)
DEVOPS-OFFLINE-34-006 | TODO | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit)
DEVOPS-OFFLINE-37-001 | TODO | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit)
DEVOPS-OFFLINE-37-002 | TODO | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit)
OFFLINE-CONTAINERS-46-001 | TODO | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline Kit Guild, Deployment Guild (ops/offline-kit)
OPS-SECRETS-02 | TODO | Add Surface.Secrets bundles (encrypted creds, manifests) to Offline Kit packaging plus verification script. Dependencies: OPS-SECRETS-02. | Offline Kit Guild, DevOps Guild (ops/offline-kit)
DEVOPS-OFFLINE-34-006 | DONE (2025-11-26) | Bundle orchestrator service container, worker SDK samples, Postgres snapshot, and dashboards into Offline Kit with manifest/signature updates. Dependencies: DEVOPS-OFFLINE-17-004. | Offline Kit Guild, Orchestrator Service Guild (ops/offline-kit)
DEVOPS-OFFLINE-37-001 | DONE (2025-11-26) | Export Center offline bundles + verification tooling (mirror artefacts, verification CLI, manifest/signature refresh, air-gap import script). Dependencies: DEVOPS-OFFLINE-34-006. | Offline Kit Guild, Exporter Service Guild (ops/offline-kit)
DEVOPS-OFFLINE-37-002 | DONE (2025-11-26) | Notifier offline packs (sample configs, template/digest packs, dry-run harness) with integrity checks and operator docs. Dependencies: DEVOPS-OFFLINE-37-001. | Offline Kit Guild, Notifications Service Guild (ops/offline-kit)
OFFLINE-CONTAINERS-46-001 | DONE (2025-11-26) | Include container air-gap bundle, verification docs, and mirrored registry instructions inside Offline Kit. | Offline Kit Guild, Deployment Guild (ops/offline-kit)
OPS-SECRETS-02 | DONE (2025-11-26) | Add Surface.Secrets bundles (encrypted creds, manifests) to Offline Kit packaging plus verification script. Dependencies: OPS-SECRETS-02. | Offline Kit Guild, DevOps Guild (ops/offline-kit)
## Execution Log
| Date (UTC) | Update | Owner |
| --- | --- | --- |
| 2025-11-26 | Wired Offline Kit packaging to include CLI binaries (release/cli), Task Runner bootstrap config, and task-pack docs; updated `test_build_offline_kit.py` to cover new artefacts. Marked CLI-PACKS-43-002 DONE. | Implementer |
| 2025-11-26 | Added container bundle pickup (release/containers/images) and mirrored registry doc copy; updated offline kit test coverage; marked OFFLINE-CONTAINERS-46-001 DONE. | Implementer |
| 2025-11-26 | Added orchestrator (service, worker SDK, postgres, dashboards), Export Center bundles, Notifier offline packs, and Surface.Secrets bundles to packaging; expanded offline kit unit test accordingly. Marked DEVOPS-OFFLINE-34-006/37-001/37-002 and OPS-SECRETS-02 DONE. | Implementer |
| 2025-11-26 | Updated Offline Kit doc (`docs/24_OFFLINE_KIT.md`) to describe newly bundled assets (CLI/task packs, orchestrator/export/notifier kits, container bundles, Surface.Secrets) and documented release-dir auto-pickup rules. | Implementer |
| 2025-11-23 | Release debug store mirrored into Offline Kit (`out/offline-kit/metadata/debug-store.json`) via `mirror_debug_store.py`. | Offline Kit Guild |

View File

@@ -696,8 +696,8 @@
| DOCS-GRAPH-24-001 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Graph Guild | docs/modules/graph | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Wait for GRAP0101 contract freeze | DOGR0101 |
| DOCS-GRAPH-24-002 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · UI Guild | docs/modules/graph | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Needs SBOM/VEX dataflow confirmation (PLLG0104) | DOGR0101 |
| DOCS-GRAPH-24-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Unblocked: SBOM join spec delivered with CARTO-GRAPH-21-002 (2025-11-17). | DOGR0101 |
| DOCS-GRAPH-24-004 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/modules/graph | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 |
| DOCS-GRAPH-24-005 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Wait for CLI samples from CLCI0109 | DOGR0101 |
| DOCS-GRAPH-24-004 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/api/graph.md; docs/api/vuln.md | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 |
| DOCS-GRAPH-24-005 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | | DOGR0101 |
| DOCS-GRAPH-24-006 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Policy Guild | docs/modules/graph | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Needs policy outputs from PLVL0102 | DOGR0101 |
| DOCS-GRAPH-24-007 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevOps Guild | docs/modules/graph | Produce `/docs/migration/graph-parity.md` with rollout plan, parity checks, fallback guidance. Dependencies: DOCS-GRAPH-24-006. | Depends on DVDO0108 deployment notes | DOGR0101 |
| DOCS-INSTALL-44-001 | BLOCKED | 2025-11-25 | SPRINT_305_docs_tasks_md_v | Docs Guild · Deployment Guild | docs/install | Publish `/docs/install/overview.md` and `/docs/install/compose-quickstart.md` with imposed rule line and copy-ready commands. | Blocked: waiting on DVPL0101 compose schema + service list/version pins | DOIS0101 |
@@ -1099,14 +1099,14 @@
| GRAPH-API-28-011 | DONE (2025-11-26) | 2025-11-26 | SPRINT_0207_0001_0001_graph | Graph API Guild | src/Graph/StellaOps.Graph.Api | Provide deployment manifests, offline kit support, API gateway integration docs, and smoke tests. Dependencies: GRAPH-API-28-010. | GRAPH-API-28-009 | GRAPI0101 |
| GRAPH-CAS-401-001 | TODO | | SPRINT_0401_0001_0001_reachability_evidence_chain | Scanner Worker Guild | `src/Scanner/StellaOps.Scanner.Worker` | Finalize richgraph schema (`richgraph-v1`), emit canonical SymbolIDs, compute graph hash (BLAKE3), and store CAS manifests under `cas://reachability/graphs/{sha256}`. Update Scanner Worker adapters + fixtures. | Depends on #1 | CASC0101 |
| GRAPH-DOCS-0001 | DONE (2025-11-05) | 2025-11-05 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Validate that graph module README/diagrams reflect the latest overlay + snapshot updates. | GRAPI0101 evidence | GRDG0101 |
| GRAPH-DOCS-0002 | TODO | 2025-11-05 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Pending DOCS-GRAPH-24-003 to add API/query doc cross-links | GRAPI0101 outputs | GRDG0101 |
| GRAPH-DOCS-0002 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Docs Guild | docs/modules/graph | Pending DOCS-GRAPH-24-003 to add API/query doc cross-links | GRAPI0101 outputs | GRDG0101 |
| GRAPH-ENG-0001 | TODO | | SPRINT_321_docs_modules_graph | Module Team | docs/modules/graph | Keep module milestones in sync with `/docs/implplan/SPRINT_141_graph.md` and related files. | GRSC0101 | GRDG0101 |
| GRAPH-INDEX-28-007 | DOING | | SPRINT_0140_0001_0001_runtime_signals | — | | Running on scanner surface mock bundle v1; will validate again once real caches drop. | — | ORGR0101 |
| GRAPH-INDEX-28-008 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Incremental update/backfill pipeline depends on 28-007 artifacts; retry/backoff plumbing sketched but blocked. | — | ORGR0101 |
| GRAPH-INDEX-28-009 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Test/fixture/chaos coverage waits on earlier jobs to exist so determinism checks have data. | — | ORGR0101 |
| GRAPH-INDEX-28-010 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Packaging/offline bundles paused until upstream graph jobs are available to embed. | — | ORGR0101 |
| GRAPH-INDEX-28-011 | TODO | 2025-11-04 | SPRINT_0207_0001_0001_graph | Graph Index Guild | src/Graph/StellaOps.Graph.Indexer | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. Dependencies: GRAPH-INDEX-28-002..006. | GRSC0101 outputs | GRIX0101 |
| GRAPH-OPS-0001 | TODO | | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 |
| GRAPH-OPS-0001 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 |
| HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild (ops/deployment) | ops/deployment | | | GRIX0101 |
| HELM-45-002 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Security Guild (ops/deployment) | ops/deployment | Add TLS/Ingress, NetworkPolicy, PodSecurityContexts, Secrets integration (external secrets), and document security posture. Dependencies: HELM-45-001. | | GRIX0101 |
| HELM-45-003 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Observability Guild (ops/deployment) | ops/deployment | Implement HPA, PDB, readiness gates, Prometheus scraping annotations, OTel configuration hooks, and upgrade hooks. Dependencies: HELM-45-002. | | GRIX0101 |
@@ -2905,7 +2905,7 @@
| DOCS-FORENSICS-53-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Timeline Indexer Guild | docs/modules/evidence-locker/forensics.md | Publish `/docs/forensics/timeline.md` with schema, event kinds, filters, query examples, and imposed rule banner. Dependencies: DOCS-FORENSICS-53-002. | Requires timeline indexer export from 055_AGIM0101 | DOEL0101 |
| DOCS-GRAPH-24-001 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Graph Guild | docs/modules/graph | Author `/docs/ui/sbom-graph-explorer.md` detailing overlays, filters, saved views, accessibility, and AOC visibility. | Wait for GRAP0101 contract freeze | DOGR0101 |
| DOCS-GRAPH-24-002 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · UI Guild | docs/modules/graph | Publish `/docs/ui/vulnerability-explorer.md` covering table usage, grouping, fix suggestions, Why drawer. Dependencies: DOCS-GRAPH-24-001. | Needs SBOM/VEX dataflow confirmation (PLLG0104) | DOGR0101 |
| DOCS-GRAPH-24-003 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Blocked on SBOM join spec from CARTO-GRAPH-21-002 | DOGR0101 |
| DOCS-GRAPH-24-003 | DONE (2025-11-26) | 2025-11-26 | SPRINT_304_docs_tasks_md_iv | Docs Guild · SBOM Guild | docs/modules/graph | Create `/docs/modules/graph/architecture-index.md` describing data model, ingestion pipeline, caches, events. Dependencies: DOCS-GRAPH-24-002. | Unblocked: SBOM join spec delivered with CARTO-GRAPH-21-002 (2025-11-17). | DOGR0101 |
| DOCS-GRAPH-24-004 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · BE-Base Guild | docs/modules/graph | Document `/docs/api/graph.md` and `/docs/api/vuln.md` avec endpoints, parameters, errors, RBAC. Dependencies: DOCS-GRAPH-24-003. | Require replay hooks from RBBN0101 | DOGR0101 |
| DOCS-GRAPH-24-005 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · DevEx/CLI Guild | docs/modules/graph | Update `/docs/modules/cli/guides/graph-and-vuln.md` covering new CLI commands, exit codes, scripting. Dependencies: DOCS-GRAPH-24-004. | Wait for CLI samples from CLCI0109 | DOGR0101 |
| DOCS-GRAPH-24-006 | TODO | | SPRINT_304_docs_tasks_md_iv | Docs Guild · Policy Guild | docs/modules/graph | Write `/docs/policy/ui-integration.md` explaining overlays, cache usage, simulator contracts. Dependencies: DOCS-GRAPH-24-005. | Needs policy outputs from PLVL0102 | DOGR0101 |
@@ -3319,7 +3319,7 @@
| GRAPH-INDEX-28-009 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Test/fixture/chaos coverage waits on earlier jobs to exist so determinism checks have data. | — | ORGR0101 |
| GRAPH-INDEX-28-010 | TODO | | SPRINT_0140_0001_0001_runtime_signals | — | | Packaging/offline bundles paused until upstream graph jobs are available to embed. | — | ORGR0101 |
| GRAPH-INDEX-28-011 | TODO | 2025-11-04 | SPRINT_0207_0001_0001_graph | Graph Index Guild | src/Graph/StellaOps.Graph.Indexer | Wire SBOM ingest runtime to emit graph snapshot artifacts, add DI factory helpers, and document Mongo/snapshot environment guidance. Dependencies: GRAPH-INDEX-28-002..006. | GRSC0101 outputs | GRIX0101 |
| GRAPH-OPS-0001 | TODO | | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 |
| GRAPH-OPS-0001 | DONE (2025-11-26) | 2025-11-26 | SPRINT_321_docs_modules_graph | Ops Guild | docs/modules/graph | Review graph observability dashboards/runbooks after the next sprint demo. | GRUI0101 | GRDG0101 |
| HELM-45-001 | TODO | | SPRINT_501_ops_deployment_i | Deployment Guild (ops/deployment) | ops/deployment | | | GRIX0101 |
| HELM-45-002 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Security Guild (ops/deployment) | ops/deployment | Add TLS/Ingress, NetworkPolicy, PodSecurityContexts, Secrets integration (external secrets), and document security posture. Dependencies: HELM-45-001. | | GRIX0101 |
| HELM-45-003 | TODO | | SPRINT_502_ops_deployment_ii | Deployment Guild, Observability Guild (ops/deployment) | ops/deployment | Implement HPA, PDB, readiness gates, Prometheus scraping annotations, OTel configuration hooks, and upgrade hooks. Dependencies: HELM-45-002. | | GRIX0101 |

View File

@@ -0,0 +1,64 @@
# Exception Governance Migration Guide
Status: Draft (2025-11-26) — aligns with Excititor VEX/exception flows and Doc task DOCS-EXC-25-007.
## Why this migration
- Retire legacy suppressions/waivers and move to policy/VEX-first exceptions with provenance.
- Provide auditable, reversible exceptions with rollout/rollback plans and notifications.
## Target state
- Exceptions are recorded as:
- VEX statements (OpenVEX) referencing findings and components.
- Policy overrides (simulator) with scope, expiry, and rationale.
- Optional authority approval attestation (DSSE) per exception bundle.
- Persistence: Excititor stores exception records with tenant, source, scope, expiry, approver, evidence links.
- Propagation:
- Graph overlays surface exception status on nodes/edges.
- Notify emits `exception.created/expired` events.
- CLI/Console consume exception status via `/exceptions` API.
## Migration steps
1) **Freeze legacy inputs**
- Disable new legacy suppressions in UI/CLI.
- Mark existing suppressions read-only; export CSV/NDJSON snapshot.
2) **Export legacy suppressions**
- Run `exc suppressions export --format ndjson --with-rationale --with-expiry`.
- Store exports with SHA256 and DSSE envelope in `exceptions/export-YYYYMMDD/`.
3) **Transform to VEX/policy overrides**
- Convert each suppression to OpenVEX statement:
- `status`: `not_affected` or `under_investigation`.
- `justification`: map legacy reason → VEX justification code.
- `impact`: optional; include nearest safe version if known.
- Generate policy override record with:
- `scope` (component, environment, service).
- `expiresAt` (carry forward or set 30/90d).
- `rationale` from legacy note.
4) **Import**
- Use Excititor `/exceptions/import` (or CLI `exc exceptions import`) to load transformed NDJSON.
- Verify import report: counts by status, rejected items, conflicts.
5) **Notify & rollout**
- Notify downstream systems: Graph overlays refresh, Scanner/Vuln Explorer respect VEX, Policy Engine caches reload.
- Announce change freeze window and rollback plan.
## Rollback plan
- Keep legacy suppressions export.
- If import fails or downstream errors:
- Disable new exception endpoints feature flag.
- Re-enable legacy suppressions (read-only → writable) temporarily.
- Investigate rejected items; re-run transform/import after fix.
## Notifications
- When migrating: send summary to tenants with counts per status and upcoming expiries.
- Ongoing: Notify events `exception.created`, `exception.expired`, `exception.rejected` with tenant + scope.
## Validation checklist
- [ ] Legacy suppressions export stored with SHA256 + DSSE.
- [ ] Transform script maps all legacy reasons → VEX justification codes.
- [ ] Import dry-run produces zero rejects or documented rejects with reasons.
- [ ] Overlay/Console shows exceptions on affected components.
- [ ] Rollback tested on staging.
## References
- Excititor module: `docs/modules/excititor/architecture.md`, `vex_observations.md`.
- Policy simulator: `docs/modules/policy/architecture.md` (POLICY-ENGINE-30-001..003).
- Graph overlays: `docs/modules/graph/architecture-index.md`.

View File

@@ -0,0 +1,82 @@
# CLI Guide · Graph & Vulnerability
Status: Draft (2025-11-26) — reflects current Graph API surface; Vuln API pending (see docs/api/vuln.md).
## Prereqs
- `stellaops-cli` built from current repo.
- Auth token with scopes: `graph:read`, `graph:query`, `graph:export` (and `vuln:read` when Vuln API lands).
- Tenant header is required for all calls (`--tenant`).
## Commands
### Search graph
```bash
stella graph search \
--tenant acme \
--kinds component \
--query "pkg:npm/" \
--limit 20 \
--format ndjson
```
Outputs NDJSON tiles (`node`, optional `cursor`).
### Query graph with budgets
```bash
stella graph query \
--tenant acme \
--kinds component \
--query widget \
--include-edges \
--budget-tiles 200 \
--budget-nodes 150 \
--budget-edges 100 \
--format ndjson
```
Returns nodes/edges/stats/cursor with cost metadata. If edge budget is exceeded, an `error` tile with `GRAPH_BUDGET_EXCEEDED` is returned.
### Paths
```bash
stella graph paths \
--tenant acme \
--source gn:acme:component:one \
--target gn:acme:component:two \
--max-depth 4 \
--include-edges \
--format ndjson
```
### Diff snapshots
```bash
stella graph diff \
--tenant acme \
--snapshot-a snapA \
--snapshot-b snapB \
--include-edges \
--format ndjson
```
### Export graph
```bash
stella graph export \
--tenant acme \
--format ndjson \
--include-edges \
--kinds component \
--query "pkg:npm/" \
--out graph-export.ndjson
```
Manifest is returned with jobId, sha256, size, and download URL; CLI downloads to `--out`.
### Vuln lookup (placeholder)
Pending Vuln Explorer API; for now use graph search/query to inspect impacted components. See `docs/api/vuln.md`.
## Exit codes
- `0` success
- `2` validation error (missing kinds/tenant/scopes)
- `3` budget exceeded (error tile received)
- `4` network/auth failure
## Tips
- Add `--format table` (when available) for quick summaries; default is NDJSON.
- Use `--cursor` from a previous call to page results.
- To sample overlays, pass `--include-overlays` (budget permitting).

View File

@@ -35,11 +35,26 @@ Graph Indexer + Graph API build the tenant-scoped knowledge graph that powers bl
- Logs/traces: structured ETL logs, query planner traces, WebGL interaction telemetry (once UI lands).
- Offline bundles: deterministic `nodes.jsonl`, `edges.jsonl`, overlay manifests + DSSE signatures, consumable by Export Center and CLI mirroring.
## Operations & runbook (Sprint 030)
- Dashboards: import `Observability/graph-api-grafana.json` (panels for latency, budget denials, overlay cache ratio, export latency). Apply tenant filter in every panel.
- Health checks: `/healthz` should be 200; search/query/paths/diff/export endpoints require `X-Stella-Tenant`, `Authorization`, and scopes (`graph:read/query/export`).
- Key metrics (new):
- `graph_tile_latency_seconds` histogram (label `route`); alert when p95 > 1.5s for 5m.
- `graph_query_budget_denied_total` counter (label `reason`); investigate spikes (>50 in 5m).
- `graph_overlay_cache_hits_total` / `graph_overlay_cache_misses_total`; watch miss ratio > 0.4 for 10m.
- `graph_export_latency_seconds` histogram (label `format`); alert when p95 > 2s for ndjson/graphml.
- Triage playbook:
- Budget denials: lower default edges/nodes budget or guide callers to request smaller scopes; verify overlay includes are truly required.
- Overlay cache misses: ensure cache TTL is ≥5m; check overlay service connectivity to Policy Engine; warm cache by replaying recent hot nodes.
- Export slowness: reduce export `Limit`, offload PNG/SVG to worker, and confirm disk I/O headroom.
- If alerts fire, capture tenant, route, cursor/budget values, and recent deploy SHA in incident note.
## Key docs & updates
- [`architecture.md`](architecture.md) — inputs, pipelines, APIs, storage choices, observability, offline handling.
- [`implementation_plan.md`](implementation_plan.md) — phased delivery roadmap, work breakdown, risks, test strategy.
- [`schema.md`](schema.md) — canonical node/edge schema and attribute dictionary (keep in sync with indexer code).
- Updates: `docs/updates/2025-10-26-scheduler-graph-jobs.md`, `docs/updates/2025-10-26-authority-graph-scopes.md`, `docs/updates/2025-10-30-devops-governance.md` for the latest decisions/dependencies.
- Index: see `architecture-index.md` for data model, ingestion pipeline, overlays/caches, events, and API/observability pointers.
## Epic alignment
- **Epic 5 SBOM Graph Explorer:** Graph Indexer, Graph API, saved queries, overlays, Console/CLI experiences, Offline Kit parity.

View File

@@ -0,0 +1,48 @@
# Graph Module · Architecture Index
Status: Draft (2025-11-26) — aligns with Sprint 0141 (Graph Indexer) and Sprint 0207 (Graph API).
## Data model (canonical)
- Nodes/edges follow `docs/modules/graph/schema.md`.
- Tenancy: every node/edge/snapshot/event carries `tenant`.
- Snapshots: immutable bundles of nodes + edges; overlays stored separately (`graph_overlays_cache`) to avoid mutating historical snapshots.
## Ingestion pipeline
1) **SBOM snapshots** from SBOM Service → Graph Indexer (`graph_snapshots`, `graph_nodes`, `graph_edges`).
2) **Advisory/VEX** from Concelier/Excititor → edges and overlay seeds.
3) **Policy overlays** from Policy Engine (POLICY-ENGINE-30-001..003) → cached per node in `graph_overlays_cache`.
4) **Runtime/signals** (future) → additional edges/attributes; stored alongside snapshot edges with tenant guard.
## Overlays & caches
- Overlay types: policy (`policy.overlay.v1`), vex (`openvex.v1`), analytics (clusters/centrality).
- Cache TTL: 510 minutes recommended; deterministic IDs (SHA256 over tenant|node|kind).
- Analytics overlays emitted as NDJSON (`overlays/clusters.ndjson`, `overlays/centrality.ndjson`) with stable ordering.
## Events & change streams
- Inputs: `sbom.snapshot.created`, `advisory.observation.updated@1`, `sbom.version.created`, `sbom.asset.updated`.
- Outputs: Graph Indexer change-stream/backfill emits node/edge upserts and overlay refresh notifications for Graph API caches.
- Downstream consumers: Scheduler (recompute jobs), Graph API (query caches), Export Center (offline bundles).
## APIs (reader surfaces)
- Graph API (Sprint 0207): `/graph/search`, `/graph/query`, `/graph/paths`, `/graph/diff`, `/graph/export`.
- Saved queries and snapshots exposed through Graph API; RBAC scopes enforced (`graph:read/query/export`).
- Health check: `/healthz` (200 when service ready).
See also `docs/api/graph.md` (pending) for full endpoint parameters and error envelopes.
## Observability (current metrics)
- `graph_tile_latency_seconds{route}`
- `graph_query_budget_denied_total{reason}`
- `graph_overlay_cache_hits_total`, `graph_overlay_cache_misses_total`
- `graph_export_latency_seconds{format}`
- Ingest/backfill metrics live in Indexer (Sprint 0141).
## Offline / bundles
- Exports: deterministic NDJSON (`nodes.jsonl`, `edges.jsonl`, overlays) with SHA256 manifest and optional DSSE.
- Air-gap friendly: no external calls; rely on cached schemas and local snapshots.
## References
- `architecture.md` — deep dive.
- `implementation_plan.md` — roadmap and risks.
- `schema.md` — canonical node/edge shapes.
- `docs/implplan/SPRINT_0141_0001_0001_graph_indexer.md` — Indexer sprint.
- `docs/implplan/SPRINT_0207_0001_0001_graph.md` — Graph API sprint.

View File

@@ -184,6 +184,45 @@ def copy_plugins_and_assets(staging_dir: Path) -> None:
copy_if_exists(REPO_ROOT / "docs" / "24_OFFLINE_KIT.md", docs_dir / "24_OFFLINE_KIT.md")
copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-collector.md", docs_dir / "telemetry-collector.md")
copy_if_exists(REPO_ROOT / "docs" / "ops" / "telemetry-storage.md", docs_dir / "telemetry-storage.md")
copy_if_exists(REPO_ROOT / "docs" / "airgap" / "mirror-bundles.md", docs_dir / "mirror-bundles.md")
def copy_cli_and_taskrunner_assets(release_dir: Path, staging_dir: Path) -> None:
"""Bundle CLI binaries, task pack docs, and Task Runner samples when available."""
cli_src = release_dir / "cli"
if cli_src.exists():
copy_if_exists(cli_src, staging_dir / "cli")
taskrunner_bootstrap = staging_dir / "bootstrap" / "task-runner"
taskrunner_bootstrap.mkdir(parents=True, exist_ok=True)
copy_if_exists(REPO_ROOT / "etc" / "task-runner.yaml.sample", taskrunner_bootstrap / "task-runner.yaml.sample")
docs_dir = staging_dir / "docs"
copy_if_exists(REPO_ROOT / "docs" / "task-packs", docs_dir / "task-packs")
copy_if_exists(REPO_ROOT / "docs" / "modules" / "taskrunner", docs_dir / "modules" / "taskrunner")
def copy_orchestrator_assets(release_dir: Path, staging_dir: Path) -> None:
"""Copy orchestrator service, worker SDK, postgres snapshot, and dashboards when present."""
mapping = {
release_dir / "orchestrator" / "service": staging_dir / "orchestrator" / "service",
release_dir / "orchestrator" / "worker-sdk": staging_dir / "orchestrator" / "worker-sdk",
release_dir / "orchestrator" / "postgres": staging_dir / "orchestrator" / "postgres",
release_dir / "orchestrator" / "dashboards": staging_dir / "orchestrator" / "dashboards",
}
for src, dest in mapping.items():
copy_if_exists(src, dest)
def copy_export_and_notifier_assets(release_dir: Path, staging_dir: Path) -> None:
"""Copy Export Center and Notifier offline bundles and tooling when present."""
copy_if_exists(release_dir / "export-center", staging_dir / "export-center")
copy_if_exists(release_dir / "notifier", staging_dir / "notifier")
def copy_surface_secrets(release_dir: Path, staging_dir: Path) -> None:
"""Include Surface.Secrets bundles and manifests if present."""
copy_if_exists(release_dir / "surface-secrets", staging_dir / "surface-secrets")
def copy_bootstrap_configs(staging_dir: Path) -> None:
@@ -267,7 +306,21 @@ def scan_files(staging_dir: Path, exclude: Optional[set[str]] = None) -> list[Or
)
)
)
return entries
return entries
def copy_container_bundles(release_dir: Path, staging_dir: Path) -> None:
"""Copy container air-gap bundles if present in the release directory."""
candidates = [release_dir / "containers", release_dir / "images"]
target_dir = staging_dir / "containers"
for root in candidates:
if not root.exists():
continue
for bundle in sorted(root.glob("**/*")):
if bundle.is_file() and bundle.suffix in {".gz", ".tar", ".tgz"}:
target_path = target_dir / bundle.relative_to(root)
target_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(bundle, target_path)
def write_offline_manifest(
@@ -372,11 +425,16 @@ def build_offline_kit(args: argparse.Namespace) -> MutableMapping[str, Any]:
if isinstance(checksums, Mapping):
release_manifest_sha = checksums.get("sha256")
copy_release_manifests(release_dir, staging_dir)
copy_release_manifests(release_dir, staging_dir)
copy_component_artifacts(manifest_data, release_dir, staging_dir)
copy_collections(manifest_data, release_dir, staging_dir)
copy_plugins_and_assets(staging_dir)
copy_bootstrap_configs(staging_dir)
copy_cli_and_taskrunner_assets(release_dir, staging_dir)
copy_container_bundles(release_dir, staging_dir)
copy_orchestrator_assets(release_dir, staging_dir)
copy_export_and_notifier_assets(release_dir, staging_dir)
copy_surface_secrets(release_dir, staging_dir)
copy_third_party_licenses(staging_dir)
package_telemetry_bundle(staging_dir)

View File

@@ -40,8 +40,42 @@ class OfflineKitBuilderTests(unittest.TestCase):
json.dump(payload, handle, indent=2)
handle.write("\n")
def _create_sample_release(self) -> None:
self.release_dir.mkdir(parents=True, exist_ok=True)
def _create_sample_release(self) -> None:
self.release_dir.mkdir(parents=True, exist_ok=True)
cli_archive = self.release_dir / "cli" / "stellaops-cli-linux-x64.tar.gz"
cli_archive.parent.mkdir(parents=True, exist_ok=True)
cli_archive.write_bytes(b"cli-bytes")
compute_sha256(cli_archive)
container_bundle = self.release_dir / "containers" / "stellaops-containers.tar.gz"
container_bundle.parent.mkdir(parents=True, exist_ok=True)
container_bundle.write_bytes(b"container-bundle")
compute_sha256(container_bundle)
orchestrator_service = self.release_dir / "orchestrator" / "service" / "orchestrator-service.tar.gz"
orchestrator_service.parent.mkdir(parents=True, exist_ok=True)
orchestrator_service.write_bytes(b"orch-service")
compute_sha256(orchestrator_service)
orchestrator_dash = self.release_dir / "orchestrator" / "dashboards" / "dash.json"
orchestrator_dash.parent.mkdir(parents=True, exist_ok=True)
orchestrator_dash.write_text("{}\n", encoding="utf-8")
export_bundle = self.release_dir / "export-center" / "export-offline-bundle.tar.gz"
export_bundle.parent.mkdir(parents=True, exist_ok=True)
export_bundle.write_bytes(b"export")
compute_sha256(export_bundle)
notifier_pack = self.release_dir / "notifier" / "notifier-offline-pack.tar.gz"
notifier_pack.parent.mkdir(parents=True, exist_ok=True)
notifier_pack.write_bytes(b"notifier")
compute_sha256(notifier_pack)
secrets_bundle = self.release_dir / "surface-secrets" / "secrets-bundle.tar.gz"
secrets_bundle.parent.mkdir(parents=True, exist_ok=True)
secrets_bundle.write_bytes(b"secrets")
compute_sha256(secrets_bundle)
sbom_path = self.release_dir / "artifacts/sboms/sample.cyclonedx.json"
sbom_path.parent.mkdir(parents=True, exist_ok=True)
@@ -114,10 +148,10 @@ class OfflineKitBuilderTests(unittest.TestCase):
encoding="utf-8",
)
manifest = OrderedDict(
(
(
"release",
manifest = OrderedDict(
(
(
"release",
OrderedDict(
(
("version", "1.0.0"),
@@ -216,9 +250,9 @@ class OfflineKitBuilderTests(unittest.TestCase):
),
)
)
write_manifest(manifest, self.release_dir)
def test_build_offline_kit(self) -> None:
write_manifest(manifest, self.release_dir)
def test_build_offline_kit(self) -> None:
args = argparse.Namespace(
version="2025.10.0",
channel="edge",
@@ -242,10 +276,34 @@ class OfflineKitBuilderTests(unittest.TestCase):
self.assertTrue((bootstrap_notify / "notify.yaml").exists())
self.assertTrue((bootstrap_notify / "notify-web.secret.example").exists())
taskrunner_bootstrap = self.staging_dir / "bootstrap" / "task-runner"
self.assertTrue((taskrunner_bootstrap / "task-runner.yaml.sample").exists())
docs_taskpacks = self.staging_dir / "docs" / "task-packs"
self.assertTrue(docs_taskpacks.exists())
self.assertTrue((self.staging_dir / "docs" / "mirror-bundles.md").exists())
containers_dir = self.staging_dir / "containers"
self.assertTrue((containers_dir / "stellaops-containers.tar.gz").exists())
orchestrator_dir = self.staging_dir / "orchestrator"
self.assertTrue((orchestrator_dir / "service" / "orchestrator-service.tar.gz").exists())
self.assertTrue((orchestrator_dir / "dashboards" / "dash.json").exists())
export_dir = self.staging_dir / "export-center"
self.assertTrue((export_dir / "export-offline-bundle.tar.gz").exists())
notifier_dir = self.staging_dir / "notifier"
self.assertTrue((notifier_dir / "notifier-offline-pack.tar.gz").exists())
secrets_dir = self.staging_dir / "surface-secrets"
self.assertTrue((secrets_dir / "secrets-bundle.tar.gz").exists())
with offline_manifest.open("r", encoding="utf-8") as handle:
manifest_data = json.load(handle)
artifacts = manifest_data["artifacts"]
self.assertTrue(any(item["name"].startswith("sboms/") for item in artifacts))
self.assertTrue(any(item["name"].startswith("cli/") for item in artifacts))
metadata_path = Path(result["metadataPath"])
data = json.loads(metadata_path.read_text(encoding="utf-8"))
@@ -258,6 +316,11 @@ class OfflineKitBuilderTests(unittest.TestCase):
self.assertTrue(any(name.startswith("sboms/sample-") for name in members))
self.assertIn("bootstrap/notify/notify.yaml", members)
self.assertIn("bootstrap/notify/notify-web.secret.example", members)
self.assertIn("containers/stellaops-containers.tar.gz", members)
self.assertIn("orchestrator/service/orchestrator-service.tar.gz", members)
self.assertIn("export-center/export-offline-bundle.tar.gz", members)
self.assertIn("notifier/notifier-offline-pack.tar.gz", members)
self.assertIn("surface-secrets/secrets-bundle.tar.gz", members)
if __name__ == "__main__":

Binary file not shown.

View File

@@ -0,0 +1 @@
dc3938d79d4e0b9a77e92dc6660391f36230b8d16c9b24b7164b6a1e6723666b telemetry-offline-bundle.tar.gz

View File

@@ -7,4 +7,4 @@
| OAS-62-001 | DOING | Populate request/response examples for top 50 endpoints, including standard error envelope. |
| OAS-62-002 | TODO | Add custom lint rules enforcing pagination, idempotency headers, naming conventions, and example coverage. |
| OAS-63-001 | TODO | Implement compatibility diff tooling comparing previous release specs; classify breaking vs additive changes. |
| OAS-63-002 | TODO | Add `/.well-known/openapi` discovery endpoint schema metadata (extensions, version info). |
| OAS-63-002 | DONE (2025-11-24) | Discovery endpoint metadata and schema extensions added; composed spec exports `/.well-known/openapi` entry. |

View File

@@ -60,7 +60,9 @@ app.MapPost("/graph/search", async (HttpContext context, GraphSearchRequest requ
return Results.Empty;
}
await foreach (var line in service.SearchAsync(tenant!, request, ct))
var tenantId = tenant!;
await foreach (var line in service.SearchAsync(tenantId, request, ct))
{
await context.Response.WriteAsync(line, ct);
await context.Response.WriteAsync("\n", ct);
@@ -113,7 +115,9 @@ app.MapPost("/graph/query", async (HttpContext context, GraphQueryRequest reques
return Results.Empty;
}
await foreach (var line in service.QueryAsync(tenant!, request, ct))
var tenantId = tenant!;
await foreach (var line in service.QueryAsync(tenantId, request, ct))
{
await context.Response.WriteAsync(line, ct);
await context.Response.WriteAsync("\n", ct);
@@ -166,7 +170,9 @@ app.MapPost("/graph/paths", async (HttpContext context, GraphPathRequest request
return Results.Empty;
}
await foreach (var line in service.FindPathsAsync(tenant!, request, ct))
var tenantId = tenant!;
await foreach (var line in service.FindPathsAsync(tenantId, request, ct))
{
await context.Response.WriteAsync(line, ct);
await context.Response.WriteAsync("\n", ct);
@@ -219,7 +225,9 @@ app.MapPost("/graph/diff", async (HttpContext context, GraphDiffRequest request,
return Results.Empty;
}
await foreach (var line in service.DiffAsync(tenant!, request, ct))
var tenantId = tenant!;
await foreach (var line in service.DiffAsync(tenantId, request, ct))
{
await context.Response.WriteAsync(line, ct);
await context.Response.WriteAsync("\n", ct);
@@ -272,7 +280,8 @@ app.MapPost("/graph/export", async (HttpContext context, GraphExportRequest requ
return Results.Empty;
}
var job = await service.StartExportAsync(tenant!, request, ct);
var tenantId = tenant!;
var job = await service.StartExportAsync(tenantId, request, ct);
var manifest = new
{
jobId = job.JobId,

View File

@@ -8,14 +8,16 @@ namespace StellaOps.Graph.Api.Services;
public sealed class InMemoryGraphDiffService : IGraphDiffService
{
private readonly InMemoryGraphRepository _repository;
private readonly IGraphMetrics _metrics;
private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public InMemoryGraphDiffService(InMemoryGraphRepository repository)
public InMemoryGraphDiffService(InMemoryGraphRepository repository, IGraphMetrics metrics)
{
_repository = repository;
_metrics = metrics;
}
public async IAsyncEnumerable<string> DiffAsync(string tenant, GraphDiffRequest request, [EnumeratorCancellation] CancellationToken ct = default)
@@ -26,6 +28,7 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
var edgeBudgetRemaining = budget.Edges ?? 10000;
var budgetRemaining = tileBudgetLimit;
var seq = 0;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var snapA = _repository.GetSnapshot(tenant, request.SnapshotA);
var snapB = _repository.GetSnapshot(tenant, request.SnapshotB);
@@ -39,6 +42,8 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
Details = new { request.SnapshotA, request.SnapshotB }
};
yield return JsonSerializer.Serialize(new TileEnvelope("error", seq++, error, Cost(tileBudgetLimit, budgetRemaining)), Options);
stopwatch.Stop();
_metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair<string, object?>("route", "/graph/diff"));
yield break;
}
@@ -49,15 +54,15 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
foreach (var added in nodesB.Values.Where(n => !nodesA.ContainsKey(n.Id)).OrderBy(n => n.Id, StringComparer.Ordinal))
{
if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; }
yield return JsonSerializer.Serialize(new TileEnvelope("node_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "nodes")); yield return tile!; yield break; }
yield return JsonSerializer.Serialize(new TileEnvelope("node_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
foreach (var removed in nodesA.Values.Where(n => !nodesB.ContainsKey(n.Id)).OrderBy(n => n.Id, StringComparer.Ordinal))
{
if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; }
yield return JsonSerializer.Serialize(new TileEnvelope("node_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "nodes")); yield return tile!; yield break; }
yield return JsonSerializer.Serialize(new TileEnvelope("node_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
foreach (var common in nodesA.Keys.Intersect(nodesB.Keys, StringComparer.Ordinal).OrderBy(k => k, StringComparer.Ordinal))
{
@@ -65,7 +70,7 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
var b = nodesB[common];
if (!AttributesEqual(a.Attributes, b.Attributes))
{
if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; }
if (!Spend(ref budgetRemaining, ref nodeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "nodes")); yield return tile!; yield break; }
var diff = new DiffTile
{
EntityType = "node",
@@ -82,13 +87,13 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
{
foreach (var added in edgesB.Values.Where(e => !edgesA.ContainsKey(e.Id)).OrderBy(e => e.Id, StringComparer.Ordinal))
{
if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; }
if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "edges")); yield return tile!; yield break; }
yield return JsonSerializer.Serialize(new TileEnvelope("edge_added", seq++, added, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
foreach (var removed in edgesA.Values.Where(e => !edgesB.ContainsKey(e.Id)).OrderBy(e => e.Id, StringComparer.Ordinal))
{
if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; }
if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "edges")); yield return tile!; yield break; }
yield return JsonSerializer.Serialize(new TileEnvelope("edge_removed", seq++, removed, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
@@ -98,12 +103,12 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
var b = edgesB[common];
if (!AttributesEqual(a.Attributes, b.Attributes))
{
if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { yield return tile!; yield break; }
var diff = new DiffTile
{
EntityType = "edge",
ChangeType = "changed",
Id = common,
if (!Spend(ref budgetRemaining, ref edgeBudgetRemaining, tileBudgetLimit, seq, out var tile)) { _metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "edges")); yield return tile!; yield break; }
var diff = new DiffTile
{
EntityType = "edge",
ChangeType = "changed",
Id = common,
Before = a,
After = b
};
@@ -126,6 +131,9 @@ public sealed class InMemoryGraphDiffService : IGraphDiffService
yield return JsonSerializer.Serialize(new TileEnvelope("stats", seq++, stats, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
stopwatch.Stop();
_metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair<string, object?>("route", "/graph/diff"));
await Task.CompletedTask;
}

View File

@@ -9,15 +9,17 @@ public sealed class InMemoryGraphPathService : IGraphPathService
{
private readonly InMemoryGraphRepository _repository;
private readonly IOverlayService _overlayService;
private readonly IGraphMetrics _metrics;
private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
};
public InMemoryGraphPathService(InMemoryGraphRepository repository, IOverlayService overlayService)
public InMemoryGraphPathService(InMemoryGraphRepository repository, IOverlayService overlayService, IGraphMetrics metrics)
{
_repository = repository;
_overlayService = overlayService;
_metrics = metrics;
}
public async IAsyncEnumerable<string> FindPathsAsync(string tenant, GraphPathRequest request, [EnumeratorCancellation] CancellationToken ct = default)
@@ -29,6 +31,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService
var edgeBudgetRemaining = budget.Edges ?? 10000;
var budgetRemaining = tileBudgetLimit;
var seq = 0;
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
var result = FindShortestPath(tenant, request, maxDepth);
@@ -42,6 +45,8 @@ public sealed class InMemoryGraphPathService : IGraphPathService
};
yield return JsonSerializer.Serialize(new TileEnvelope("error", seq++, error, Cost(tileBudgetLimit, budgetRemaining)), Options);
stopwatch.Stop();
_metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair<string, object?>("route", "/graph/paths"));
yield break;
}
@@ -58,6 +63,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService
{
if (budgetRemaining <= 0 || nodeBudgetRemaining <= 0)
{
_metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "tiles"));
yield return BudgetExceeded(tileBudgetLimit, budgetRemaining, seq++);
yield break;
}
@@ -75,6 +81,7 @@ public sealed class InMemoryGraphPathService : IGraphPathService
{
if (budgetRemaining <= 0 || edgeBudgetRemaining <= 0)
{
_metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "tiles"));
yield return BudgetExceeded(tileBudgetLimit, budgetRemaining, seq++);
yield break;
}
@@ -93,6 +100,9 @@ public sealed class InMemoryGraphPathService : IGraphPathService
yield return JsonSerializer.Serialize(new TileEnvelope("stats", seq++, stats, Cost(tileBudgetLimit, budgetRemaining)), Options);
}
stopwatch.Stop();
_metrics.QueryLatencySeconds.Record(stopwatch.Elapsed.TotalSeconds, new KeyValuePair<string, object?>("route", "/graph/paths"));
await Task.CompletedTask;
}

View File

@@ -49,21 +49,17 @@ public sealed class InMemoryGraphQueryService : IGraphQueryService
var cursorOffset = CursorCodec.Decode(request.Cursor);
var (nodes, edges) = _repository.QueryGraph(tenant, request);
if (nodes.Count > nodeBudgetLimit)
{
_metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "nodes"));
yield return BuildBudgetError(cacheKey, "nodes", nodeBudgetLimit, nodes.Count, edges.Count, budget);
yield break;
}
if (request.IncludeEdges && edges.Count > edgeBudgetLimit)
{
_metrics.BudgetDenied.Add(1, new KeyValuePair<string, object?>("reason", "edges"));
var error = new ErrorResponse
{
Error = "GRAPH_BUDGET_EXCEEDED",
Message = $"Query exceeded edge budget (edges>{edgeBudgetLimit}).",
Details = new { nodes = nodes.Count, edges = edges.Count, budget }
};
var errorLine = JsonSerializer.Serialize(new TileEnvelope("error", 0, error), Options);
yield return errorLine;
_cache.Set(cacheKey, new[] { errorLine }, new MemoryCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(2)
});
yield return BuildBudgetError(cacheKey, "edges", edgeBudgetLimit, nodes.Count, edges.Count, budget);
yield break;
}
@@ -206,4 +202,20 @@ public sealed class InMemoryGraphQueryService : IGraphQueryService
private static CostBudget Cost(int limit, int remainingBudget) =>
new(limit, remainingBudget - 1, limit - (remainingBudget - 1));
private string BuildBudgetError(string cacheKey, string exceeded, int budgetLimit, int nodesCount, int edgesCount, GraphQueryBudget budget)
{
var error = new ErrorResponse
{
Error = "GRAPH_BUDGET_EXCEEDED",
Message = $"Query exceeded {exceeded} budget ({exceeded}>{budgetLimit}).",
Details = new { nodes = nodesCount, edges = edgesCount, budget }
};
var errorLine = JsonSerializer.Serialize(new TileEnvelope("error", 0, error), Options);
_cache.Set(cacheKey, new[] { errorLine }, new MemoryCacheEntryOptions
{
AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(2)
});
return errorLine;
}
}

View File

@@ -44,7 +44,7 @@ namespace StellaOps.Graph.Api.Services;
}
// Always return a fresh copy so we can inject a single explain trace without polluting cache.
var overlays = new Dictionary<string, OverlayPayload>(cachedBase, StringComparer.Ordinal);
var overlays = new Dictionary<string, OverlayPayload>(cachedBase!, StringComparer.Ordinal);
if (sampleExplain && !explainEmitted)
{

View File

@@ -11,7 +11,8 @@ public class DiffServiceTests
public async Task DiffAsync_EmitsAddedRemovedChangedAndStats()
{
var repo = new InMemoryGraphRepository();
var service = new InMemoryGraphDiffService(repo);
var metrics = new GraphMetrics();
var service = new InMemoryGraphDiffService(repo, metrics);
var request = new GraphDiffRequest
{
@@ -37,7 +38,8 @@ public class DiffServiceTests
public async Task DiffAsync_WhenSnapshotMissing_ReturnsError()
{
var repo = new InMemoryGraphRepository();
var service = new InMemoryGraphDiffService(repo);
var metrics = new GraphMetrics();
var service = new InMemoryGraphDiffService(repo, metrics);
var request = new GraphDiffRequest
{

View File

@@ -13,8 +13,9 @@ public class PathServiceTests
{
var repo = new InMemoryGraphRepository();
var cache = new MemoryCache(new MemoryCacheOptions());
var overlays = new InMemoryOverlayService(cache);
var service = new InMemoryGraphPathService(repo, overlays);
var metrics = new GraphMetrics();
var overlays = new InMemoryOverlayService(cache, metrics);
var service = new InMemoryGraphPathService(repo, overlays, metrics);
var request = new GraphPathRequest
{
@@ -39,8 +40,9 @@ public class PathServiceTests
{
var repo = new InMemoryGraphRepository();
var cache = new MemoryCache(new MemoryCacheOptions());
var overlays = new InMemoryOverlayService(cache);
var service = new InMemoryGraphPathService(repo, overlays);
var metrics = new GraphMetrics();
var overlays = new InMemoryOverlayService(cache, metrics);
var service = new InMemoryGraphPathService(repo, overlays, metrics);
var request = new GraphPathRequest
{

View File

@@ -107,8 +107,9 @@ namespace StellaOps.Graph.Api.Tests;
private static InMemoryGraphQueryService CreateService(InMemoryGraphRepository? repository = null)
{
var cache = new MemoryCache(new MemoryCacheOptions());
var overlays = new InMemoryOverlayService(cache);
return new InMemoryGraphQueryService(repository ?? new InMemoryGraphRepository(), cache, overlays);
var cache = new MemoryCache(new MemoryCacheOptions());
var metrics = new GraphMetrics();
var overlays = new InMemoryOverlayService(cache, metrics);
return new InMemoryGraphQueryService(repository ?? new InMemoryGraphRepository(), cache, overlays, metrics);
}
}

View File

@@ -121,8 +121,9 @@ public class SearchServiceTests
}, Array.Empty<EdgeTile>());
var cache = new MemoryCache(new MemoryCacheOptions());
var overlays = new InMemoryOverlayService(cache);
var service = new InMemoryGraphQueryService(repo, cache, overlays);
var metrics = new GraphMetrics();
var overlays = new InMemoryOverlayService(cache, metrics);
var service = new InMemoryGraphQueryService(repo, cache, overlays, metrics);
var request = new GraphQueryRequest
{
Kinds = new[] { "component" },
@@ -154,8 +155,9 @@ public class SearchServiceTests
});
var cache = new MemoryCache(new MemoryCacheOptions());
var overlays = new InMemoryOverlayService(cache);
var service = new InMemoryGraphQueryService(repo, cache, overlays);
var metrics = new GraphMetrics();
var overlays = new InMemoryOverlayService(cache, metrics);
var service = new InMemoryGraphQueryService(repo, cache, overlays, metrics);
var request = new GraphQueryRequest
{
Kinds = new[] { "component" },

View File

@@ -0,0 +1,12 @@
{
"solution": {
"path": "StellaOps.Policy.sln",
"projects": [
"__Libraries/StellaOps.Policy/StellaOps.Policy.csproj",
"__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj",
"../__Libraries/StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj",
"../__Libraries/StellaOps.Configuration/StellaOps.Configuration.csproj",
"../__Libraries/StellaOps.Cryptography/StellaOps.Cryptography.csproj"
]
}
}

View File

@@ -0,0 +1,9 @@
{
"solution": {
"path": "StellaOps.Policy.sln",
"projects": [
"__Libraries/StellaOps.Policy/StellaOps.Policy.csproj",
"__Tests/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj"
]
}
}

View File

@@ -10,6 +10,7 @@ using System.Text.Json.Serialization;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.Lang;
using StellaOps.Scanner.Core.Contracts;
using StellaOps.Scanner.Core.Entropy;
using StellaOps.Scanner.EntryTrace;
using StellaOps.Scanner.EntryTrace.Serialization;
using StellaOps.Scanner.Surface.Env;
@@ -185,6 +186,41 @@ internal sealed class SurfaceManifestStageExecutor : IScanStageExecutor
RegisterArtifact: true));
}
if (context.Analysis.TryGet<EntropyReport>(ScanAnalysisKeys.EntropyReport, out var entropyReport) && entropyReport is not null)
{
var json = JsonSerializer.Serialize(entropyReport, JsonOptions);
payloads.Add(new SurfaceManifestPayload(
ArtifactDocumentType.SurfaceObservation,
ArtifactDocumentFormat.ObservationJson,
Kind: "entropy.report",
MediaType: "application/json",
Content: Encoding.UTF8.GetBytes(json),
View: "entropy",
Metadata: new Dictionary<string, string>
{
["imageDigest"] = entropyReport.ImageDigest,
["layerDigest"] = entropyReport.LayerDigest,
["opaqueRatio"] = entropyReport.ImageOpaqueRatio.ToString("0.####", CultureInfoInvariant)
}));
}
if (context.Analysis.TryGet<EntropyLayerSummary>(ScanAnalysisKeys.EntropyLayerSummary, out var entropySummary) && entropySummary is not null)
{
var json = JsonSerializer.Serialize(entropySummary, JsonOptions);
payloads.Add(new SurfaceManifestPayload(
ArtifactDocumentType.SurfaceObservation,
ArtifactDocumentFormat.ObservationJson,
Kind: "entropy.layer-summary",
MediaType: "application/json",
Content: Encoding.UTF8.GetBytes(json),
View: "entropy",
Metadata: new Dictionary<string, string>
{
["layerDigest"] = entropySummary.LayerDigest,
["opaqueRatio"] = entropySummary.OpaqueRatio.ToString("0.####", CultureInfoInvariant)
}));
}
return payloads;
}

View File

@@ -28,4 +28,8 @@ public static class ScanAnalysisKeys
public const string ReachabilityUnionGraph = "analysis.reachability.union.graph";
public const string ReachabilityUnionCas = "analysis.reachability.union.cas";
public const string FileEntries = "analysis.files.entries";
public const string EntropyReport = "analysis.entropy.report";
public const string EntropyLayerSummary = "analysis.entropy.layer.summary";
}

View File

@@ -0,0 +1,71 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.Runtime;
namespace StellaOps.Zastava.Observer.Backend;
internal interface IRuntimeFactsClient
{
Task PublishAsync(RuntimeFactsPublishRequest request, CancellationToken cancellationToken);
}
internal sealed class RuntimeFactsClient : IRuntimeFactsClient
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
};
private readonly HttpClient httpClient;
private readonly IOptionsMonitor<ZastavaObserverOptions> observerOptions;
private readonly ILogger<RuntimeFactsClient> logger;
public RuntimeFactsClient(
HttpClient httpClient,
IOptionsMonitor<ZastavaObserverOptions> observerOptions,
ILogger<RuntimeFactsClient> logger)
{
this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public async Task PublishAsync(RuntimeFactsPublishRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var reachability = observerOptions.CurrentValue.Reachability ?? new ReachabilityRuntimeOptions();
var endpoint = reachability.Endpoint;
if (string.IsNullOrWhiteSpace(endpoint))
{
throw new RuntimeFactsException("Reachability endpoint is not configured.");
}
using var message = new HttpRequestMessage(HttpMethod.Post, endpoint);
if (!string.IsNullOrWhiteSpace(reachability.AnalysisId))
{
message.Headers.TryAddWithoutValidation("X-Analysis-Id", reachability.AnalysisId);
}
var json = JsonSerializer.Serialize(request, SerializerOptions);
message.Content = new StringContent(json, Encoding.UTF8, "application/json");
message.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
using var response = await httpClient.SendAsync(message, cancellationToken).ConfigureAwait(false);
if (response.IsSuccessStatusCode)
{
return;
}
var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
logger.LogWarning("Runtime facts publish failed with status {Status}: {Body}", (int)response.StatusCode, body);
throw new RuntimeFactsException($"Runtime facts publish failed with status {(int)response.StatusCode}.");
}
}

View File

@@ -0,0 +1,55 @@
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Zastava.Observer.Configuration;
/// <summary>
/// Configuration for emitting runtime reachability facts to Signals.
/// </summary>
public sealed class ReachabilityRuntimeOptions
{
/// <summary>
/// Enables runtime reachability fact publishing when true.
/// </summary>
public bool Enabled { get; set; }
/// <summary>
/// Signals endpoint that accepts runtime facts (JSON or NDJSON).
/// </summary>
[Required(AllowEmptyStrings = false)]
public string Endpoint { get; set; } = "https://signals.internal/signals/runtime-facts";
/// <summary>
/// Required callgraph identifier used to correlate runtime facts with static graphs.
/// </summary>
[Required(AllowEmptyStrings = false)]
public string CallgraphId { get; set; } = string.Empty;
/// <summary>
/// Optional analysis identifier forwarded as X-Analysis-Id.
/// </summary>
public string? AnalysisId { get; set; }
/// <summary>
/// Maximum number of facts sent in a single publish attempt.
/// </summary>
[Range(1, 5000)]
public int BatchSize { get; set; } = 1000;
/// <summary>
/// Maximum delay (seconds) before flushing a partially filled batch.
/// </summary>
[Range(typeof(double), "0.1", "30")]
public double FlushIntervalSeconds { get; set; } = 2;
/// <summary>
/// Optional subject fallback when image data is missing.
/// </summary>
public string? SubjectScanId { get; set; }
public string? SubjectImageDigest { get; set; }
public string? SubjectComponent { get; set; }
public string? SubjectVersion { get; set; }
}

View File

@@ -7,8 +7,8 @@ namespace StellaOps.Zastava.Observer.Configuration;
/// <summary>
/// Observer-specific configuration applied on top of the shared runtime options.
/// </summary>
public sealed class ZastavaObserverOptions
{
public sealed class ZastavaObserverOptions
{
public const string SectionName = "zastava:observer";
private const string DefaultContainerdSocket = "unix:///run/containerd/containerd.sock";
@@ -126,6 +126,12 @@ public sealed class ZastavaObserverOptions
/// </summary>
[Range(1, 128)]
public int MaxEntrypointArguments { get; set; } = 32;
/// <summary>
/// Runtime reachability fact publishing configuration.
/// </summary>
[Required]
public ReachabilityRuntimeOptions Reachability { get; init; } = new();
}
public sealed class ZastavaObserverBackendOptions

View File

@@ -87,6 +87,14 @@ public static class ObserverServiceCollectionExtensions
client.Timeout = TimeSpan.FromSeconds(Math.Clamp(backend.RequestTimeoutSeconds, 1, 120));
});
services.AddHttpClient<IRuntimeFactsClient, RuntimeFactsClient>()
.ConfigureHttpClient((provider, client) =>
{
var optionsMonitor = provider.GetRequiredService<IOptionsMonitor<ZastavaObserverOptions>>();
var observer = optionsMonitor.CurrentValue;
client.Timeout = TimeSpan.FromSeconds(Math.Clamp(observer.Backend.RequestTimeoutSeconds, 1, 120));
});
services.TryAddEnumerable(ServiceDescriptor.Singleton<IPostConfigureOptions<ZastavaRuntimeOptions>, ObserverRuntimeOptionsPostConfigure>());
// Surface environment + cache/manifest/secrets wiring

View File

@@ -0,0 +1,357 @@
using System;
using System.Security.Cryptography;
using System.Text;
using System.Linq;
using System.Collections.Generic;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
namespace StellaOps.Zastava.Observer.Runtime;
internal static class RuntimeFactsBuilder
{
public static RuntimeFactsPublishRequest? Build(
IReadOnlyCollection<RuntimeEventEnvelope> envelopes,
ReachabilityRuntimeOptions options)
{
ArgumentNullException.ThrowIfNull(envelopes);
ArgumentNullException.ThrowIfNull(options);
if (!options.Enabled || envelopes.Count == 0)
{
return null;
}
if (string.IsNullOrWhiteSpace(options.CallgraphId))
{
return null;
}
var facts = new List<RuntimeFactEventPayload>(envelopes.Count);
ReachabilitySubjectPayload? subject = null;
foreach (var envelope in envelopes)
{
var fact = TryBuildFact(envelope);
if (fact is null)
{
continue;
}
facts.Add(fact);
subject ??= BuildSubject(envelope, options);
}
if (facts.Count == 0)
{
return null;
}
subject ??= BuildFallbackSubject(options);
if (subject is null)
{
return null;
}
facts.Sort(CompareFacts);
return new RuntimeFactsPublishRequest
{
CallgraphId = options.CallgraphId.Trim(),
Subject = subject,
Events = facts
};
}
private static RuntimeFactEventPayload? TryBuildFact(RuntimeEventEnvelope envelope)
{
ArgumentNullException.ThrowIfNull(envelope);
var evt = envelope.Event;
if (evt.Kind != RuntimeEventKind.ContainerStart || evt.Process is null)
{
return null;
}
var symbolId = BuildSymbolId(evt.Process);
if (string.IsNullOrWhiteSpace(symbolId))
{
return null;
}
var fact = new RuntimeFactEventPayload
{
SymbolId = symbolId,
CodeId = Normalize(evt.Process.BuildId),
BuildId = Normalize(evt.Process.BuildId),
LoaderBase = null,
Purl = null,
SymbolDigest = null,
HitCount = 1,
ObservedAt = evt.When,
ProcessId = evt.Process.Pid,
ProcessName = ResolveProcessName(evt.Process.Entrypoint),
ContainerId = evt.Workload.ContainerId,
Metadata = BuildMetadata(evt.Process)
};
return fact;
}
private static string? BuildSymbolId(RuntimeProcess process)
{
if (!string.IsNullOrWhiteSpace(process.BuildId))
{
return $"sym:binary:{process.BuildId.Trim().ToLowerInvariant()}";
}
var trace = process.EntryTrace?.FirstOrDefault(t =>
!string.IsNullOrWhiteSpace(t.Target) || !string.IsNullOrWhiteSpace(t.File));
var seed = trace?.Target ?? trace?.File ?? process.Entrypoint.FirstOrDefault();
if (string.IsNullOrWhiteSpace(seed))
{
return null;
}
var stable = ComputeStableFragment(seed);
return $"sym:shell:{stable}";
}
private static ReachabilitySubjectPayload? BuildSubject(RuntimeEventEnvelope envelope, ReachabilityRuntimeOptions options)
{
var workload = envelope.Event.Workload;
var imageRef = workload.ImageRef;
var digest = ExtractImageDigest(imageRef);
var (component, version) = ExtractComponentAndVersion(imageRef);
digest ??= Normalize(options.SubjectImageDigest);
component ??= Normalize(options.SubjectComponent);
version ??= Normalize(options.SubjectVersion);
var scanId = Normalize(options.SubjectScanId);
if (string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(scanId)
&& (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version)))
{
return null;
}
return new ReachabilitySubjectPayload
{
ScanId = scanId,
ImageDigest = digest,
Component = component,
Version = version
};
}
private static ReachabilitySubjectPayload? BuildFallbackSubject(ReachabilityRuntimeOptions options)
{
var digest = Normalize(options.SubjectImageDigest);
var component = Normalize(options.SubjectComponent);
var version = Normalize(options.SubjectVersion);
var scanId = Normalize(options.SubjectScanId);
if (string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(scanId)
&& (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version)))
{
return null;
}
return new ReachabilitySubjectPayload
{
ScanId = scanId,
ImageDigest = digest,
Component = component,
Version = version
};
}
private static Dictionary<string, string?>? BuildMetadata(RuntimeProcess process)
{
if (process.EntryTrace is null || process.EntryTrace.Count == 0)
{
return null;
}
var sb = new StringBuilder();
foreach (var trace in process.EntryTrace)
{
if (trace is null)
{
continue;
}
var op = string.IsNullOrWhiteSpace(trace.Op) ? "exec" : trace.Op;
var target = trace.Target ?? trace.File;
if (string.IsNullOrWhiteSpace(target))
{
continue;
}
if (sb.Length > 0)
{
sb.Append(" | ");
}
sb.Append(op).Append(':').Append(target);
}
if (sb.Length == 0)
{
return null;
}
return new Dictionary<string, string?>(StringComparer.Ordinal)
{
["entryTrace"] = sb.ToString()
};
}
private static string? ExtractImageDigest(string? imageRef)
{
if (string.IsNullOrWhiteSpace(imageRef))
{
return null;
}
var digestStart = imageRef.IndexOf("sha256:", StringComparison.OrdinalIgnoreCase);
if (digestStart >= 0)
{
return imageRef[digestStart..].Trim();
}
var atIndex = imageRef.IndexOf('@');
if (atIndex > 0 && atIndex + 1 < imageRef.Length)
{
return imageRef[(atIndex + 1)..].Trim();
}
return null;
}
private static (string? Component, string? Version) ExtractComponentAndVersion(string? imageRef)
{
if (string.IsNullOrWhiteSpace(imageRef))
{
return (null, null);
}
var lastSlash = imageRef.LastIndexOf('/');
var lastColon = imageRef.LastIndexOf(':');
if (lastColon < 0 || lastColon < lastSlash)
{
return (null, null);
}
var component = imageRef[(lastSlash + 1)..lastColon];
var version = imageRef[(lastColon + 1)..];
if (string.IsNullOrWhiteSpace(component) || string.IsNullOrWhiteSpace(version))
{
return (null, null);
}
return (component.Trim(), version.Trim());
}
private static string? ResolveProcessName(IReadOnlyList<string>? entrypoint)
{
if (entrypoint is null || entrypoint.Count == 0)
{
return null;
}
var first = entrypoint[0];
if (string.IsNullOrWhiteSpace(first))
{
return null;
}
var lastSlash = first.LastIndexOf('/');
return lastSlash >= 0 ? first[(lastSlash + 1)..] : first;
}
private static string ComputeStableFragment(string seed)
{
var normalized = seed.Trim().ToLowerInvariant();
var bytes = Encoding.UTF8.GetBytes(normalized);
Span<byte> hash = stackalloc byte[32];
_ = SHA256.TryHashData(bytes, hash, out _);
return Convert.ToHexString(hash[..16]).ToLowerInvariant();
}
private static int CompareFacts(RuntimeFactEventPayload left, RuntimeFactEventPayload right)
{
var timeComparison = Nullable.Compare(left.ObservedAt, right.ObservedAt);
if (timeComparison != 0)
{
return timeComparison;
}
return string.Compare(left.SymbolId, right.SymbolId, StringComparison.Ordinal);
}
private static string? Normalize(string? value)
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
}
internal sealed class RuntimeFactsPublishRequest
{
public ReachabilitySubjectPayload Subject { get; set; } = new();
public string CallgraphId { get; set; } = string.Empty;
public List<RuntimeFactEventPayload> Events { get; set; } = new();
}
internal sealed class ReachabilitySubjectPayload
{
public string? ScanId { get; set; }
public string? ImageDigest { get; set; }
public string? Component { get; set; }
public string? Version { get; set; }
}
internal sealed class RuntimeFactEventPayload
{
public string SymbolId { get; set; } = string.Empty;
public string? CodeId { get; set; }
public string? SymbolDigest { get; set; }
public string? Purl { get; set; }
public string? BuildId { get; set; }
public string? LoaderBase { get; set; }
public int? ProcessId { get; set; }
public string? ProcessName { get; set; }
public string? SocketAddress { get; set; }
public string? ContainerId { get; set; }
public string? EvidenceUri { get; set; }
public int HitCount { get; set; } = 1;
public DateTimeOffset? ObservedAt { get; set; }
public Dictionary<string, string?>? Metadata { get; set; }
}
internal sealed class RuntimeFactsException : Exception
{
public RuntimeFactsException(string message, Exception? innerException = null) : base(message, innerException)
{
}
}

View File

@@ -13,6 +13,7 @@ internal sealed class RuntimeEventDispatchService : BackgroundService
{
private readonly IRuntimeEventBuffer buffer;
private readonly IRuntimeEventsClient eventsClient;
private readonly IRuntimeFactsClient runtimeFactsClient;
private readonly IOptionsMonitor<ZastavaObserverOptions> observerOptions;
private readonly TimeProvider timeProvider;
private readonly ILogger<RuntimeEventDispatchService> logger;
@@ -20,12 +21,14 @@ internal sealed class RuntimeEventDispatchService : BackgroundService
public RuntimeEventDispatchService(
IRuntimeEventBuffer buffer,
IRuntimeEventsClient eventsClient,
IRuntimeFactsClient runtimeFactsClient,
IOptionsMonitor<ZastavaObserverOptions> observerOptions,
TimeProvider timeProvider,
ILogger<RuntimeEventDispatchService> logger)
{
this.buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
this.eventsClient = eventsClient ?? throw new ArgumentNullException(nameof(eventsClient));
this.runtimeFactsClient = runtimeFactsClient ?? throw new ArgumentNullException(nameof(runtimeFactsClient));
this.observerOptions = observerOptions ?? throw new ArgumentNullException(nameof(observerOptions));
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -116,14 +119,17 @@ internal sealed class RuntimeEventDispatchService : BackgroundService
return;
}
var request = new RuntimeEventsIngestRequest
{
BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}",
Events = batch.Select(item => item.Envelope).ToArray()
};
try
{
var envelopes = batch.Select(item => item.Envelope).ToArray();
var factsPublished = await TryPublishRuntimeFactsAsync(envelopes, cancellationToken).ConfigureAwait(false);
var request = new RuntimeEventsIngestRequest
{
BatchId = $"obs-{timeProvider.GetUtcNow():yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}",
Events = envelopes
};
var result = await eventsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false);
if (result.Success)
{
@@ -132,10 +138,11 @@ internal sealed class RuntimeEventDispatchService : BackgroundService
await item.CompleteAsync().ConfigureAwait(false);
}
logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}).",
logger.LogInformation("Runtime events batch published (batchId={BatchId}, accepted={Accepted}, duplicates={Duplicates}, runtimeFacts={FactsPublished}).",
request.BatchId,
result.Accepted,
result.Duplicates);
result.Duplicates,
factsPublished);
}
else if (result.RateLimited)
{
@@ -166,6 +173,38 @@ internal sealed class RuntimeEventDispatchService : BackgroundService
}
}
private async Task<bool> TryPublishRuntimeFactsAsync(RuntimeEventEnvelope[] envelopes, CancellationToken cancellationToken)
{
if (envelopes.Length == 0)
{
return false;
}
var options = observerOptions.CurrentValue.Reachability;
var request = RuntimeFactsBuilder.Build(envelopes, options);
if (request is null)
{
return false;
}
try
{
await runtimeFactsClient.PublishAsync(request, cancellationToken).ConfigureAwait(false);
logger.LogDebug("Published {Count} runtime facts (callgraphId={CallgraphId}).", request.Events.Count, request.CallgraphId);
return true;
}
catch (RuntimeFactsException ex) when (!cancellationToken.IsCancellationRequested)
{
logger.LogWarning(ex, "Runtime facts publish failed; batch will be retried.");
throw;
}
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
{
logger.LogWarning(ex, "Runtime facts publish encountered an unexpected error; batch will be retried.");
throw;
}
}
private async Task RequeueBatchAsync(IEnumerable<RuntimeEventBufferItem> batch, CancellationToken cancellationToken)
{
foreach (var item in batch)

View File

@@ -0,0 +1,93 @@
using System;
using System.Collections.Generic;
using StellaOps.Zastava.Core.Contracts;
using StellaOps.Zastava.Observer.Configuration;
using StellaOps.Zastava.Observer.Runtime;
using Xunit;
namespace StellaOps.Zastava.Observer.Tests.Runtime;
public sealed class RuntimeFactsBuilderTests
{
[Fact]
public void Build_UsesBuildIdAndDigest()
{
var options = new ReachabilityRuntimeOptions
{
Enabled = true,
CallgraphId = "cg-001"
};
var runtimeEvent = CreateRuntimeEvent(
imageRef: "ghcr.io/example/api@sha256:deadbeef",
buildId: "beadfeed",
when: DateTimeOffset.Parse("2025-11-26T12:00:00Z"));
var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent);
var result = RuntimeFactsBuilder.Build(new[] { envelope }, options);
Assert.NotNull(result);
Assert.Equal("cg-001", result!.CallgraphId);
Assert.Equal("sha256:deadbeef", result.Subject.ImageDigest);
Assert.Single(result.Events);
Assert.StartsWith("sym:binary:beadfeed", result.Events[0].SymbolId, StringComparison.Ordinal);
}
[Fact]
public void Build_ParsesComponentAndVersion_WhenTagPresent()
{
var options = new ReachabilityRuntimeOptions
{
Enabled = true,
CallgraphId = "cg-002"
};
var runtimeEvent = CreateRuntimeEvent(
imageRef: "registry.local/team/web:1.2.3",
buildId: null,
when: DateTimeOffset.Parse("2025-11-26T12:01:00Z"));
var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent);
var result = RuntimeFactsBuilder.Build(new[] { envelope }, options);
Assert.NotNull(result);
Assert.Equal("web", result!.Subject.Component);
Assert.Equal("1.2.3", result.Subject.Version);
Assert.Single(result.Events);
Assert.StartsWith("sym:shell:", result.Events[0].SymbolId, StringComparison.Ordinal);
}
private static RuntimeEvent CreateRuntimeEvent(string imageRef, string? buildId, DateTimeOffset when)
{
var process = new RuntimeProcess
{
Pid = 1234,
Entrypoint = new[] { "/entrypoint.sh", "-c", "echo hi" },
EntryTrace = new[] { new RuntimeEntryTrace { File = "/entrypoint.sh", Op = "exec", Target = "/entrypoint.sh" } },
BuildId = buildId
};
return new RuntimeEvent
{
EventId = "evt-1",
When = when,
Kind = RuntimeEventKind.ContainerStart,
Tenant = "tenant-a",
Node = "node-a",
Runtime = new RuntimeEngine { Engine = "containerd", Version = "1.7" },
Workload = new RuntimeWorkload
{
Platform = "kubernetes",
Namespace = "default",
Pod = "pod-a",
Container = "api",
ContainerId = "containerd://abc",
ImageRef = imageRef
},
Process = process,
LoadedLibraries = Array.Empty<RuntimeLoadedLibrary>(),
Evidence = Array.Empty<RuntimeEvidence>()
};
}
}