diff --git a/CLAUDE.md b/CLAUDE.md index 29d64c641..0bf85c3c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,6 +82,8 @@ The codebase follows a monorepo pattern with modules under `src/`: | Authority | `src/Authority/` | Authentication, authorization, OAuth/OIDC, DPoP | | Gateway | `src/Gateway/` | API gateway with routing and transport abstraction | | Router | `src/Router/` | Transport-agnostic messaging (TCP/TLS/UDP/RabbitMQ/Valkey) | +| Platform | `src/Platform/` | Console backend aggregation service (health, quotas, search) | +| Registry | `src/Registry/` | Token service for container registry authentication | | **Data Ingestion** | | | | Concelier | `src/Concelier/` | Vulnerability advisory ingestion and merge engine | | Excititor | `src/Excititor/` | VEX document ingestion and export | @@ -89,13 +91,14 @@ The codebase follows a monorepo pattern with modules under `src/`: | VexHub | `src/VexHub/` | VEX distribution and exchange hub | | IssuerDirectory | `src/IssuerDirectory/` | Issuer trust registry (CSAF publishers) | | Feedser | `src/Feedser/` | Evidence collection library for backport detection | -| Mirror | `src/Mirror/` | Vulnerability feed mirror and distribution | +| Mirror | `src/Concelier/__Libraries/` | Vulnerability feed mirror connector (Concelier plugin) | | **Scanning & Analysis** | | | | Scanner | `src/Scanner/` | Container scanning with SBOM generation (11 language analyzers) | | BinaryIndex | `src/BinaryIndex/` | Binary identity extraction and fingerprinting | | AdvisoryAI | `src/AdvisoryAI/` | AI-assisted advisory analysis | | ReachGraph | `src/ReachGraph/` | Reachability graph service | | Symbols | `src/Symbols/` | Symbol resolution and debug information | +| Cartographer | `src/Cartographer/` | Dependency graph mapping and visualization | | **Artifacts & Evidence** | | | | Attestor | `src/Attestor/` | in-toto/DSSE attestation generation | | Signer | `src/Signer/` | Cryptographic signing operations | @@ -108,6 +111,7 @@ The codebase follows a monorepo pattern with modules under `src/`: | RiskEngine | `src/RiskEngine/` | Risk scoring runtime with pluggable providers | | VulnExplorer | `src/VulnExplorer/` | Vulnerability exploration and triage UI backend | | Unknowns | `src/Unknowns/` | Unknown component and symbol tracking | +| Findings | `src/Findings/` | Findings ledger service for vulnerability tracking | | **Operations** | | | | Scheduler | `src/Scheduler/` | Job scheduling and queue management | | Orchestrator | `src/Orchestrator/` | Workflow orchestration and task coordination | @@ -121,7 +125,7 @@ The codebase follows a monorepo pattern with modules under `src/`: | CLI | `src/Cli/` | Command-line interface (Native AOT) | | Zastava | `src/Zastava/` | Container registry webhook observer | | Web | `src/Web/` | Angular 17 frontend SPA | -| API | `src/Api/` | OpenAPI contracts and governance | +| Integrations | `src/Integrations/` | External system integrations web service | | **Infrastructure** | | | | Cryptography | `src/Cryptography/` | Crypto plugins (FIPS, eIDAS, GOST, SM, PQ) | | Telemetry | `src/Telemetry/` | OpenTelemetry traces, metrics, logging | @@ -129,8 +133,12 @@ The codebase follows a monorepo pattern with modules under `src/`: | Signals | `src/Signals/` | Runtime signal collection and correlation | | AirGap | `src/AirGap/` | Air-gapped deployment support | | AOC | `src/Aoc/` | Append-Only Contract enforcement (Roslyn analyzers) | +| SmRemote | `src/SmRemote/` | SM2/SM3/SM4 cryptographic remote service | +| **Development Tools** | | | +| Tools | `src/Tools/` | Development utilities (fixture updater, smoke tests, validators) | +| Bench | `src/Bench/` | Performance benchmark infrastructure | -> **Note:** See `docs/modules//architecture.md` for detailed module dossiers. +> **Note:** See `docs/modules//architecture.md` for detailed module dossiers. Some entries in `docs/modules/` are cross-cutting concepts (snapshot, triage) or shared libraries (provcache) rather than standalone modules. ### Code Organization Patterns diff --git a/Directory.Build.rsp b/Directory.Build.rsp deleted file mode 100644 index e09b43ead..000000000 --- a/Directory.Build.rsp +++ /dev/null @@ -1,4 +0,0 @@ -/nowarn:CA2022 -/p:DisableWorkloadResolver=true -/p:RestoreAdditionalProjectFallbackFolders= -/p:RestoreFallbackFolders= diff --git a/NuGet.config b/NuGet.config index f3b3be7f9..80969174d 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,6 +9,14 @@ + + + + + + + + diff --git a/docs/API_CLI_REFERENCE.md b/docs/API_CLI_REFERENCE.md index ef4fcb1bc..febad9fa4 100755 --- a/docs/API_CLI_REFERENCE.md +++ b/docs/API_CLI_REFERENCE.md @@ -21,8 +21,8 @@ Detailed references live under `docs/api/` and `docs/modules/cli/`. | Download aggregated OpenAPI via CLI | `docs/modules/cli/guides/commands/api.md` | | CLI command reference (by command group) | `docs/modules/cli/guides/commands/` | | CLI authentication and tokens | `docs/modules/cli/guides/commands/auth.md` | -| CLI Concelier job triggers (`db`) | `docs/modules/cli/guides/commands/db.md`, `docs/10_CONCELIER_CLI_QUICKSTART.md` | -| CLI offline/air-gap workflows | `docs/modules/cli/guides/commands/offline.md`, `docs/24_OFFLINE_KIT.md` | +| CLI Concelier job triggers (`db`) | `docs/modules/cli/guides/commands/db.md`, `docs/CONCELIER_CLI_QUICKSTART.md` | +| CLI offline/air-gap workflows | `docs/modules/cli/guides/commands/offline.md`, `docs/OFFLINE_KIT.md` | | CLI reachability workflow | `docs/modules/cli/guides/commands/reachability.md` | | CLI vulnerability workflow | `docs/modules/cli/guides/commands/vuln.md` | | Vuln Explorer OpenAPI (v1) | `docs/modules/vuln-explorer/openapi/vuln-explorer.v1.yaml` | diff --git a/docs/INDEX.md b/docs/INDEX.md new file mode 100644 index 000000000..6674c5a1f --- /dev/null +++ b/docs/INDEX.md @@ -0,0 +1,257 @@ +# StellaOps Documentation Index + +> **Master index of all StellaOps documentation.** +> Last updated: 2026-01-05 (Post-consolidation) + +This index provides a complete map of documentation organized by audience and topic. The documentation follows a two-level hierarchy: +- **Canonical guides** (`docs/*.md`) - High-level entry points +- **Detailed references** (`docs/**/*`) - Module dossiers, API contracts, runbooks + +--- + +## Quick Navigation by Audience + +| Audience | Start Here | +|----------|------------| +| **New Users** | [quickstart.md](quickstart.md), [overview.md](overview.md) | +| **Developers** | [DEVELOPER_ONBOARDING.md](DEVELOPER_ONBOARDING.md), [CODING_STANDARDS.md](CODING_STANDARDS.md) | +| **Architects** | [ARCHITECTURE_OVERVIEW.md](ARCHITECTURE_OVERVIEW.md), [ARCHITECTURE_REFERENCE.md](ARCHITECTURE_REFERENCE.md) | +| **Operators/SREs** | [SECURITY_HARDENING_GUIDE.md](SECURITY_HARDENING_GUIDE.md), [OFFLINE_KIT.md](OFFLINE_KIT.md) | +| **Plugin Developers** | [PLUGIN_SDK_GUIDE.md](PLUGIN_SDK_GUIDE.md), [dev/](dev/) | + +--- + +## Canonical Guides (docs/*.md) + +### Getting Started +| Document | Purpose | +|----------|---------| +| [README.md](README.md) | Documentation overview and navigation | +| [overview.md](overview.md) | 2-minute product summary | +| [quickstart.md](quickstart.md) | First scan walkthrough | +| [DEVELOPER_ONBOARDING.md](DEVELOPER_ONBOARDING.md) | Developer setup guide | +| [CONCELIER_CLI_QUICKSTART.md](CONCELIER_CLI_QUICKSTART.md) | Advisory ingestion quickstart | + +### Architecture +| Document | Purpose | +|----------|---------| +| [ARCHITECTURE_OVERVIEW.md](ARCHITECTURE_OVERVIEW.md) | 10-minute architecture tour | +| [ARCHITECTURE_REFERENCE.md](ARCHITECTURE_REFERENCE.md) | Full architecture index/map | +| [technical/architecture/](technical/architecture/) | Detailed architecture views | + +### Features & Capabilities +| Document | Purpose | +|----------|---------| +| [key-features.md](key-features.md) | Capability cards with evidence | +| [FEATURE_MATRIX.md](FEATURE_MATRIX.md) | Tier-by-tier feature availability | +| [full-features-list.md](full-features-list.md) | Complete capability catalog | + +### Operations & Security +| Document | Purpose | +|----------|---------| +| [SECURITY_HARDENING_GUIDE.md](SECURITY_HARDENING_GUIDE.md) | Deployment security guide | +| [SECURITY_POLICY.md](SECURITY_POLICY.md) | Security incident policy | +| [OFFLINE_KIT.md](OFFLINE_KIT.md) | Air-gapped operation guide | +| [UI_GUIDE.md](UI_GUIDE.md) | Console operator guide | + +### Development +| Document | Purpose | +|----------|---------| +| [CODING_STANDARDS.md](CODING_STANDARDS.md) | Code quality rules | +| [PLUGIN_SDK_GUIDE.md](PLUGIN_SDK_GUIDE.md) | Plugin development guide | +| [VEX_CONSENSUS_GUIDE.md](VEX_CONSENSUS_GUIDE.md) | VEX consensus and trust | + +### Reference +| Document | Purpose | +|----------|---------| +| [API_CLI_REFERENCE.md](API_CLI_REFERENCE.md) | API and CLI reference hub | +| [GLOSSARY.md](GLOSSARY.md) | Platform terminology | +| [ROADMAP.md](ROADMAP.md) | Product roadmap | + +--- + +## Module Documentation (docs/modules/) + +Module dossiers contain architecture, operations, and API documentation per component. + +> **Naming Convention:** Module directories use kebab-case (e.g., `binary-index`, `sbom-service`) + +### Core Platform +| Module | Directory | Description | +|--------|-----------|-------------| +| Authority | [authority/](modules/authority/) | OAuth/OIDC, DPoP authentication | +| Gateway | [gateway/](modules/gateway/) | API gateway, routing | +| Router | [router/](modules/router/) | Transport-agnostic messaging | +| Platform | [platform/](modules/platform/) | Console backend aggregation | + +### Data Ingestion +| Module | Directory | Description | +|--------|-----------|-------------| +| Concelier | [concelier/](modules/concelier/) | Advisory ingestion | +| Excititor | [excititor/](modules/excititor/) | VEX document ingestion | +| VexLens | [vex-lens/](modules/vex-lens/) | VEX consensus computation | +| VexHub | [vex-hub/](modules/vex-hub/) | VEX distribution hub | +| IssuerDirectory | [issuer-directory/](modules/issuer-directory/) | Issuer trust registry | +| Feedser | [feedser/](modules/feedser/) | Backport detection evidence | + +### Scanning & Analysis +| Module | Directory | Description | +|--------|-----------|-------------| +| Scanner | [scanner/](modules/scanner/) | Container scanning, SBOM generation | +| BinaryIndex | [binary-index/](modules/binary-index/) | Binary fingerprinting | +| AdvisoryAI | [advisory-ai/](modules/advisory-ai/) | AI-assisted analysis | +| Symbols | [symbols/](modules/symbols/) | Symbol resolution | +| ReachGraph | [reach-graph/](modules/reach-graph/) | Reachability graphs | + +### Artifacts & Evidence +| Module | Directory | Description | +|--------|-----------|-------------| +| Attestor | [attestor/](modules/attestor/) | DSSE/in-toto attestations | +| Signer | [signer/](modules/signer/) | Cryptographic signing | +| SbomService | [sbom-service/](modules/sbom-service/) | SBOM storage, lineage | +| EvidenceLocker | [evidence-locker/](modules/evidence-locker/) | Sealed evidence storage | +| ExportCenter | [export-center/](modules/export-center/) | Batch export | +| Provenance | [provenance/](modules/provenance/) | SLSA attestation | + +### Policy & Risk +| Module | Directory | Description | +|--------|-----------|-------------| +| Policy | [policy/](modules/policy/) | K4 lattice policy engine | +| RiskEngine | [risk-engine/](modules/risk-engine/) | Risk scoring | +| VulnExplorer | [vuln-explorer/](modules/vuln-explorer/) | Vulnerability triage | +| Unknowns | [unknowns/](modules/unknowns/) | Unknown component tracking | +| FindingsLedger | [findings-ledger/](modules/findings-ledger/) | Findings tracking | + +### Operations +| Module | Directory | Description | +|--------|-----------|-------------| +| Scheduler | [scheduler/](modules/scheduler/) | Job scheduling | +| Orchestrator | [orchestrator/](modules/orchestrator/) | Workflow orchestration | +| TaskRunner | [taskrunner/](modules/taskrunner/) | Task pack execution | +| Notify | [notify/](modules/notify/) | Notifications | +| Notifier | [notifier/](modules/notifier/) | Notifications Studio | +| PacksRegistry | [packs-registry/](modules/packs-registry/) | Task packs registry | +| TimelineIndexer | [timeline-indexer/](modules/timeline-indexer/) | Event indexing | +| Replay | [replay/](modules/replay/) | Deterministic replay | + +### Integration +| Module | Directory | Description | +|--------|-----------|-------------| +| CLI | [cli/](modules/cli/) | Command-line interface | +| Zastava | [zastava/](modules/zastava/) | Registry webhooks | +| Web/UI | [ui/](modules/ui/), [web/](modules/web/) | Frontend SPA | + +### Infrastructure +| Module | Directory | Description | +|--------|-----------|-------------| +| Cryptography | [cryptography/](modules/cryptography/) | Crypto profiles | +| Telemetry | [telemetry/](modules/telemetry/) | Observability | +| Graph | [graph/](modules/graph/) | Call graph structures | +| Signals | [signals/](modules/signals/) | Runtime signals | +| AirGap | [airgap/](modules/airgap/) | Air-gap support | +| AOC | [aoc/](modules/aoc/) | Append-Only Contract | + +### Cross-Cutting Concepts +| Concept | Directory | Description | +|---------|-----------|-------------| +| Snapshot | [snapshot/](modules/snapshot/) | Point-in-time captures | +| Triage | [triage/](modules/triage/) | Vulnerability triage workflows | +| Provcache | [prov-cache/](modules/prov-cache/) | Provenance cache (library) | +| Benchmark | [benchmark/](modules/benchmark/) | Competitive benchmarking | +| Bench | [bench/](modules/bench/) | Performance benchmarks | + +--- + +## Specialized Documentation Areas + +### API Documentation +| Area | Path | Description | +|------|------|-------------| +| API Overview | [api/overview.md](api/overview.md) | API conventions | +| Gateway APIs | [api/gateway/](api/gateway/) | Gateway endpoints | +| Console APIs | [api/console/](api/console/) | Console endpoints | +| Signal Contracts | [api/signals/](api/signals/) | Signal contracts | + +### Air-Gap Operations +| Area | Path | Description | +|------|------|-------------| +| Overview | [airgap/overview.md](airgap/overview.md) | Air-gap overview | +| Operations | [airgap/operations.md](airgap/operations.md) | Operational guides | +| Bundles | [airgap/](airgap/) | Bundle formats | + +### Database +| Area | Path | Description | +|------|------|-------------| +| Specification | [db/SPECIFICATION.md](db/SPECIFICATION.md) | Database spec | +| Migrations | [db/tasks/](db/tasks/) | Migration phases | +| Schemas | [db/schemas/](db/schemas/) | Schema definitions | + +### CLI Reference +| Area | Path | Description | +|------|------|-------------| +| Command Reference | [cli/command-reference.md](cli/command-reference.md) | Complete CLI reference | +| Admin Commands | [cli/admin-reference.md](cli/admin-reference.md) | Admin commands | +| Crypto Commands | [cli/crypto-commands.md](cli/crypto-commands.md) | Crypto operations | + +### End-to-End Flows +| Area | Path | Description | +|------|------|-------------| +| Flow Index | [flows/README.md](flows/README.md) | All workflow flows | +| Scan Flow | [flows/02-scan-submission-flow.md](flows/02-scan-submission-flow.md) | Scan submission | +| Policy Flow | [flows/04-policy-evaluation-flow.md](flows/04-policy-evaluation-flow.md) | Policy evaluation | +| CI/CD Flow | [flows/10-cicd-gate-flow.md](flows/10-cicd-gate-flow.md) | CI/CD gating | + +### Technical Deep Dives +| Area | Path | Description | +|------|------|-------------| +| Architecture Index | [technical/architecture/](technical/architecture/) | Architecture views | +| User Flows | [technical/architecture/user-flows.md](technical/architecture/user-flows.md) | UML diagrams | +| Module Matrix | [technical/architecture/module-matrix.md](technical/architecture/module-matrix.md) | 46-module matrix | + +### Contracts & ADRs +| Area | Path | Description | +|------|------|-------------| +| Contracts | [contracts/](contracts/) | Technical contracts | +| ADRs | [adr/](adr/) | Architecture decisions | + +### Development Guides +| Area | Path | Description | +|------|------|-------------| +| Plugin Development | [dev/](dev/) | Plugin guides & templates | +| Scanner Engine | [dev/scanning-engine.md](dev/scanning-engine.md) | Scanner internals | + +### Benchmarks & Testing +| Area | Path | Description | +|------|------|-------------| +| Benchmarks | [benchmarks/](benchmarks/) | Performance & accuracy | +| Ground Truth | [benchmarks/ground-truth-corpus.md](benchmarks/ground-truth-corpus.md) | Test datasets | + +### Risk Scoring +| Area | Path | Description | +|------|------|-------------| +| Risk Samples | [risk/samples/](risk/samples/) | Risk scoring examples | + +--- + +## Implementation Planning + +| Area | Path | Description | +|------|------|-------------| +| Sprint Files | [implplan/](implplan/) | Active implementation sprints | +| Archived Sprints | [../docs-archived/implplan/](../docs-archived/implplan/) | Completed sprints | + +--- + +## External References + +- **CLAUDE.md** (repository root) - Claude Code instructions and module table +- **src/__Tests/AGENTS.md** - Test infrastructure guidance +- **Module AGENTS.md files** - Per-module development instructions + +--- + +## Changelog + +| Date | Change | +|------|--------| +| 2026-01-05 | Created index; renamed module directories to kebab-case; updated CLAUDE.md with missing modules | diff --git a/docs/INSTALL_GUIDE.md b/docs/INSTALL_GUIDE.md index 171a11e86..939968ad9 100755 --- a/docs/INSTALL_GUIDE.md +++ b/docs/INSTALL_GUIDE.md @@ -38,7 +38,7 @@ docker compose --env-file airgap.env -f docker-compose.airgap.yaml up -d ``` For offline bundles, imports, and update workflows, use: -- `docs/24_OFFLINE_KIT.md` +- `docs/OFFLINE_KIT.md` - `docs/airgap/overview.md` - `docs/airgap/importer.md` - `docs/airgap/controller.md` diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 183598f2f..4149ce0d2 100755 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,5 +30,5 @@ This repository is the source of truth for StellaOps direction. The roadmap is e - `docs/03_VISION.md` - `docs/04_FEATURE_MATRIX.md` - `docs/40_ARCHITECTURE_OVERVIEW.md` -- `docs/24_OFFLINE_KIT.md` +- `docs/OFFLINE_KIT.md` - `docs/key-features.md` diff --git a/docs/SYSTEM_REQUIREMENTS_SPEC.md b/docs/SYSTEM_REQUIREMENTS_SPEC.md index 6c3e6953a..ef8e445fd 100755 --- a/docs/SYSTEM_REQUIREMENTS_SPEC.md +++ b/docs/SYSTEM_REQUIREMENTS_SPEC.md @@ -20,9 +20,9 @@ Scope includes core platform, CLI, UI, quota layer, and plug‑in host; commerci * [overview.md](overview.md) – market gap & problem statement * [03_VISION.md](03_VISION.md) – north‑star, KPIs, quarterly themes -* [07_HIGH_LEVEL_ARCHITECTURE.md](07_HIGH_LEVEL_ARCHITECTURE.md) – context & data flow diagrams +* [ARCHITECTURE_OVERVIEW.md](ARCHITECTURE_OVERVIEW.md) – context & data flow diagrams * [modules/platform/architecture-overview.md](modules/platform/architecture-overview.md) – component APIs & plug‑in contracts -* [09_API_CLI_REFERENCE.md](09_API_CLI_REFERENCE.md) – REST & CLI surface +* [API_CLI_REFERENCE.md](API_CLI_REFERENCE.md) – REST & CLI surface --- diff --git a/docs/UI_GUIDE.md b/docs/UI_GUIDE.md index c0f0934fa..aa0ee5314 100755 --- a/docs/UI_GUIDE.md +++ b/docs/UI_GUIDE.md @@ -17,7 +17,7 @@ Out of scope: API shapes, schema details, and UI component implementation. - **Tenant context:** most views are tenant-scoped; switching tenants changes what evidence you see and what actions you can take. - **Evidence-linked decisions:** verdicts (ship/block/needs-exception) should link to the SBOM facts, advisory/VEX observations, reachability proofs, and policy explanations that justify them. -- **Effective VEX:** the platform computes an effective status using issuer trust and policy rules, without rewriting upstream VEX (see `docs/16_VEX_CONSENSUS_GUIDE.md`). +- **Effective VEX:** the platform computes an effective status using issuer trust and policy rules, without rewriting upstream VEX (see `docs/VEX_CONSENSUS_GUIDE.md`). - **Snapshots and staleness:** offline sites operate on snapshots; the Console should surface snapshot identity and freshness rather than hide it. ## Workspaces (Navigation) @@ -46,21 +46,21 @@ The Console is organized into workspaces. Names vary slightly by build, but the 3. Record a triage action (assign/comment/ack/mute/exception request) with justification. 4. Export an evidence bundle when review, escalation, or offline verification is required. -See `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` for the conceptual model and determinism requirements. +See `docs/VULNERABILITY_EXPLORER_GUIDE.md` for the conceptual model and determinism requirements. ### Review VEX Conflicts and Issuer Trust - Use **Advisories & VEX** to see which providers contributed statements, whether signatures verified, and where conflicts exist. - The Console should not silently hide conflicts; it should show what disagrees and why, and how policy resolved it. -See `docs/16_VEX_CONSENSUS_GUIDE.md` for the underlying concepts. +See `docs/VEX_CONSENSUS_GUIDE.md` for the underlying concepts. ### Export and Verify Evidence Bundles - Exports are intended to be portable and verifiable (audits, incident response, air-gap review). - Expect deterministic ordering, UTC timestamps, and hash manifests. -See `docs/24_OFFLINE_KIT.md` for packaging and offline verification workflows. +See `docs/OFFLINE_KIT.md` for packaging and offline verification workflows. ## Offline / Air-Gap Expectations @@ -99,9 +99,9 @@ UX and interaction contracts: ## Related Docs -- `docs/16_VEX_CONSENSUS_GUIDE.md` -- `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` -- `docs/24_OFFLINE_KIT.md` +- `docs/VEX_CONSENSUS_GUIDE.md` +- `docs/VULNERABILITY_EXPLORER_GUIDE.md` +- `docs/OFFLINE_KIT.md` - `docs/cli-vs-ui-parity.md` - `docs/architecture/console-admin-rbac.md` - `docs/architecture/console-branding.md` diff --git a/docs/VEX_CONSENSUS_GUIDE.md b/docs/VEX_CONSENSUS_GUIDE.md index 61bae68dd..3affe7b9d 100644 --- a/docs/VEX_CONSENSUS_GUIDE.md +++ b/docs/VEX_CONSENSUS_GUIDE.md @@ -79,7 +79,7 @@ The Console uses these concepts to keep VEX explainable: - Conflicts are displayed as conflicts (what disagrees and why), not silently resolved in the UI. - The effective VEX status shown in triage views links back to underlying observations/linksets and the policy explanation. -See `docs/15_UI_GUIDE.md` for the operator workflow perspective. +See `docs/UI_GUIDE.md` for the operator workflow perspective. ## Offline / Air-Gap Operation @@ -91,5 +91,5 @@ See `docs/15_UI_GUIDE.md` for the operator workflow perspective. - `docs/modules/excititor/architecture.md` - `docs/modules/vex-lens/architecture.md` -- `docs/07_HIGH_LEVEL_ARCHITECTURE.md` -- `docs/24_OFFLINE_KIT.md` +- `docs/ARCHITECTURE_OVERVIEW.md` +- `docs/OFFLINE_KIT.md` diff --git a/docs/VULNERABILITY_EXPLORER_GUIDE.md b/docs/VULNERABILITY_EXPLORER_GUIDE.md index 072ad6722..580446503 100644 --- a/docs/VULNERABILITY_EXPLORER_GUIDE.md +++ b/docs/VULNERABILITY_EXPLORER_GUIDE.md @@ -84,13 +84,13 @@ The Explorer is designed to be replayable and tamper-evident: - **Console UI:** findings list + triage case view; evidence drawers; export/download flows. - **Policy engine:** produces explainability traces and gates actions (for example, exception workflows). - **Graph/Reachability:** overlays and evidence slices for reachable vs not reachable decisions where available. -- **VEX Lens / Excititor:** issuer trust, provenance, linksets, and effective status (see `docs/16_VEX_CONSENSUS_GUIDE.md`). +- **VEX Lens / Excititor:** issuer trust, provenance, linksets, and effective status (see `docs/VEX_CONSENSUS_GUIDE.md`). ## Related Docs -- `docs/15_UI_GUIDE.md` -- `docs/16_VEX_CONSENSUS_GUIDE.md` +- `docs/UI_GUIDE.md` +- `docs/VEX_CONSENSUS_GUIDE.md` - `docs/modules/vuln-explorer/architecture.md` - `docs/modules/findings-ledger/schema.md` - `docs/modules/findings-ledger/merkle-anchor-policy.md` -- `docs/07_HIGH_LEVEL_ARCHITECTURE.md` +- `docs/ARCHITECTURE_OVERVIEW.md` diff --git a/docs/accessibility.md b/docs/accessibility.md index faef8f38f..bec90fefc 100644 --- a/docs/accessibility.md +++ b/docs/accessibility.md @@ -28,7 +28,7 @@ This guide defines the StellaOps Console accessibility baseline: keyboard intera | Area | Action | macOS | Windows/Linux | Notes | | --- | --- | --- | --- | --- | | Findings | Search within explain | `Cmd+/` | `Ctrl+/` | Only when explain drawer is open. | -| SBOM Explorer | Toggle overlays | `Cmd+G` | `Ctrl+G` | Persists per session (see `docs/15_UI_GUIDE.md`). | +| SBOM Explorer | Toggle overlays | `Cmd+G` | `Ctrl+G` | Persists per session (see `docs/UI_GUIDE.md`). | | Advisories & VEX | Focus provider chips | `Cmd+Alt+F` | `Ctrl+Alt+F` | Moves focus to provider chip row. | | Runs | Refresh stream state | `Cmd+R` | `Ctrl+R` | Soft refresh; no full reload. | | Policies | Save draft | `Cmd+S` | `Ctrl+S` | Requires edit scope. | @@ -59,7 +59,7 @@ This guide defines the StellaOps Console accessibility baseline: keyboard intera ## References -- `docs/15_UI_GUIDE.md` +- `docs/UI_GUIDE.md` - `docs/cli-vs-ui-parity.md` - `docs/observability/ui-telemetry.md` - `docs/security/console-security.md` diff --git a/docs/advisory-ai/console.md b/docs/advisory-ai/console.md index ab6a4dd9c..248c077a3 100644 --- a/docs/advisory-ai/console.md +++ b/docs/advisory-ai/console.md @@ -176,7 +176,7 @@ Violations: 1. **Volume readiness** – confirm the RWX volume (`/var/lib/advisory-ai/{queue,plans,outputs}`) is mounted; the console should poll `/api/v1/advisory-ai/health` and surface “Queue not available” if the worker is offline. 2. **Cached responses** – when running air-gapped, highlight that only cached plans/responses are available by showing the `planFromCache` badge plus the `generatedAtUtc` timestamp. 3. **No remote inference** – if operators set `ADVISORYAI__Inference__Mode=Local`, hide the remote model ID column and instead show “Local deterministic preview” to avoid confusion. -4. **Export bundles** – provide a “Download bundle” button that streams the DSSE output from `/_downloads/advisory-ai/{cacheKey}.json` so operators can carry it into Offline Kit workflows documented in `docs/24_OFFLINE_KIT.md`. While staging endpoints are pending, reuse the Evidence Bundle v1 sample at `docs/samples/evidence-bundle/evidence-bundle-v1.tar.gz` (hash in `evidence-bundle-v1.tar.gz.sha256`) to validate wiring and any optional visual captures. +4. **Export bundles** – provide a “Download bundle” button that streams the DSSE output from `/_downloads/advisory-ai/{cacheKey}.json` so operators can carry it into Offline Kit workflows documented in `docs/OFFLINE_KIT.md`. While staging endpoints are pending, reuse the Evidence Bundle v1 sample at `docs/samples/evidence-bundle/evidence-bundle-v1.tar.gz` (hash in `evidence-bundle-v1.tar.gz.sha256`) to validate wiring and any optional visual captures. ## 6. Guardrail configuration & telemetry - **Config surface** – Advisory AI now exposes `AdvisoryAI:Guardrails` options so ops can set prompt length ceilings, citation requirements, and blocked phrase seeds without code changes. Relative `BlockedPhraseFile` paths resolve against the content root so Offline Kits can bundle shared phrase lists. diff --git a/docs/airgap/advisory-implementation-roadmap.md b/docs/airgap/advisory-implementation-roadmap.md index 28da7f7d6..9ecbd9f26 100644 --- a/docs/airgap/advisory-implementation-roadmap.md +++ b/docs/airgap/advisory-implementation-roadmap.md @@ -335,5 +335,5 @@ src/Authority/ - [14-Dec-2025 Offline and Air-Gap Technical Reference](../product-advisories/14-Dec-2025%20-%20Offline%20and%20Air-Gap%20Technical%20Reference.md) - [Air-Gap Mode Playbook](./airgap-mode.md) -- [Offline Kit Documentation](../24_OFFLINE_KIT.md) +- [Offline Kit Documentation](../OFFLINE_KIT.md) - [Importer](./importer.md) diff --git a/docs/airgap/offline-bundle-format.md b/docs/airgap/offline-bundle-format.md index 4f7434bec..aca88deec 100644 --- a/docs/airgap/offline-bundle-format.md +++ b/docs/airgap/offline-bundle-format.md @@ -209,5 +209,5 @@ stellaops alert bundle import --file ./bundles/alert-123.stella.bundle.tgz - [Evidence Bundle Envelope](./evidence-bundle-envelope.md) - [DSSE Signing Guide](./dsse-signing.md) -- [Offline Kit Guide](../24_OFFLINE_KIT.md) +- [Offline Kit Guide](../OFFLINE_KIT.md) - [API Reference](../api/evidence-decision-api.openapi.yaml) diff --git a/docs/airgap/offline-parity-verification.md b/docs/airgap/offline-parity-verification.md index 36b92ae54..438991ed5 100644 --- a/docs/airgap/offline-parity-verification.md +++ b/docs/airgap/offline-parity-verification.md @@ -506,7 +506,7 @@ groups: ## 9. REFERENCES -- [Offline Update Kit (OUK)](../24_OFFLINE_KIT.md) +- [Offline Update Kit (OUK)](../OFFLINE_KIT.md) - [Offline and Air-Gap Technical Reference](../product-advisories/14-Dec-2025%20-%20Offline%20and%20Air-Gap%20Technical%20Reference.md) - [Determinism and Reproducibility Technical Reference](../product-advisories/14-Dec-2025%20-%20Determinism%20and%20Reproducibility%20Technical%20Reference.md) - [Determinism CI Harness](../modules/scanner/design/determinism-ci-harness.md) diff --git a/docs/airgap/risk-bundles.md b/docs/airgap/risk-bundles.md index 25a683cbd..1f83c65ef 100644 --- a/docs/airgap/risk-bundles.md +++ b/docs/airgap/risk-bundles.md @@ -383,7 +383,7 @@ stella config set risk-bundle.trust-root /path/to/pubkey.pem ## Related Documentation -- [Offline Update Kit](../24_OFFLINE_KIT.md) - Complete offline kit documentation +- [Offline Update Kit](../OFFLINE_KIT.md) - Complete offline kit documentation - [Mirror Bundles](./mirror-bundles.md) - OCI artifact bundles for air-gap - [Provider Matrix](../modules/export-center/operations/risk-bundle-provider-matrix.md) - Detailed provider specifications - [ExportCenter Architecture](../modules/export-center/architecture.md) - Export service design diff --git a/docs/airgap/smart-diff-airgap-workflows.md b/docs/airgap/smart-diff-airgap-workflows.md index 1d0010660..874b51d71 100644 --- a/docs/airgap/smart-diff-airgap-workflows.md +++ b/docs/airgap/smart-diff-airgap-workflows.md @@ -282,7 +282,7 @@ stellaops offline kit verify \ ## Related Documentation -- [Offline Kit Guide](../24_OFFLINE_KIT.md) +- [Offline Kit Guide](../OFFLINE_KIT.md) - [Smart-Diff CLI](../cli/smart-diff-cli.md) - [Smart-Diff types](../api/smart-diff-types.md) - [Determinism gates](../testing/determinism-gates.md) diff --git a/docs/airgap/symbol-bundles.md b/docs/airgap/symbol-bundles.md index 3f04fcccb..44fcaaa8e 100644 --- a/docs/airgap/symbol-bundles.md +++ b/docs/airgap/symbol-bundles.md @@ -310,7 +310,7 @@ For offline verification: ## Related Documentation -- [Offline Kit Guide](../24_OFFLINE_KIT.md) +- [Offline Kit Guide](../OFFLINE_KIT.md) - [Symbol Server Architecture](../modules/scanner/architecture.md) - [DSSE Signing Guide](../modules/signer/architecture.md) - [Rekor Integration](../modules/attestor/architecture.md) diff --git a/docs/airgap/triage-airgap-workflows.md b/docs/airgap/triage-airgap-workflows.md index 98e1d17af..9b2610385 100644 --- a/docs/airgap/triage-airgap-workflows.md +++ b/docs/airgap/triage-airgap-workflows.md @@ -361,7 +361,7 @@ stellaops triage import-decisions \ ## Related Documentation -- [Offline Kit Guide](../24_OFFLINE_KIT.md) -- [Vulnerability Explorer guide](../20_VULNERABILITY_EXPLORER_GUIDE.md) +- [Offline Kit Guide](../OFFLINE_KIT.md) +- [Vulnerability Explorer guide](../VULNERABILITY_EXPLORER_GUIDE.md) - [Triage contract](../api/triage.contract.v1.md) - [Console accessibility](../accessibility.md) diff --git a/docs/architecture/console-admin-rbac.md b/docs/architecture/console-admin-rbac.md index 0ee588f32..2105e87da 100644 --- a/docs/architecture/console-admin-rbac.md +++ b/docs/architecture/console-admin-rbac.md @@ -232,5 +232,5 @@ Scopes: `authority:tokens.read|revoke`, `authority:audit.read` ## 9. References - `docs/modules/authority/architecture.md` - `docs/modules/ui/architecture.md` -- `docs/15_UI_GUIDE.md` +- `docs/UI_GUIDE.md` - `docs/contracts/web-gateway-tenant-rbac.md` diff --git a/docs/architecture/console-branding.md b/docs/architecture/console-branding.md index 5b6a6daed..92047a13b 100644 --- a/docs/architecture/console-branding.md +++ b/docs/architecture/console-branding.md @@ -65,7 +65,7 @@ If Authority is unreachable, the UI uses the static defaults. - Console shows last applied branding hash for verification. ## 8. References -- `docs/15_UI_GUIDE.md` +- `docs/UI_GUIDE.md` - `docs/modules/ui/architecture.md` - `docs/modules/authority/architecture.md` diff --git a/docs/architecture/enforcement-rules.md b/docs/architecture/enforcement-rules.md index c7ad1a292..da1a023bc 100644 --- a/docs/architecture/enforcement-rules.md +++ b/docs/architecture/enforcement-rules.md @@ -109,7 +109,7 @@ dotnet test tests/architecture/StellaOps.Architecture.Tests --logger "console;ve ## References -- [docs/07_HIGH_LEVEL_ARCHITECTURE.md](../07_HIGH_LEVEL_ARCHITECTURE.md) – High-level architecture overview +- [docs/07_HIGH_LEVEL_ARCHITECTURE.md](../ARCHITECTURE_OVERVIEW.md) – High-level architecture overview - [docs/modules/scanner/architecture.md](../modules/scanner/architecture.md) – Scanner module architecture (lattice engine details) - [AGENTS.md](../../AGENTS.md) – Project-wide agent guidelines and module boundaries - [NetArchTest Documentation](https://github.com/BenMorris/NetArchTest) diff --git a/docs/architecture/signal-contract-mapping.md b/docs/architecture/signal-contract-mapping.md index 61798fc74..31df1c4f9 100644 --- a/docs/architecture/signal-contract-mapping.md +++ b/docs/architecture/signal-contract-mapping.md @@ -958,7 +958,7 @@ While StellaOps uses domain-specific entity names instead of generic "Signal-X" - in-toto attestation framework: https://github.com/in-toto/attestation ### StellaOps Documentation -- `docs/07_HIGH_LEVEL_ARCHITECTURE.md` +- `docs/ARCHITECTURE_OVERVIEW.md` - `docs/modules/scanner/architecture.md` - `docs/modules/attestor/transparency.md` - `docs/contracts/witness-v1.md` diff --git a/docs/ci/sarif-integration.md b/docs/ci/sarif-integration.md index fff0057b8..445d56d99 100644 --- a/docs/ci/sarif-integration.md +++ b/docs/ci/sarif-integration.md @@ -246,5 +246,5 @@ If SARIF contains no results: - [Smart-Diff Detection Rules](../modules/scanner/smart-diff-rules.md) - [Scanner API Reference](../api/scanner-api.md) -- [CLI Reference](../09_API_CLI_REFERENCE.md) +- [CLI Reference](../API_CLI_REFERENCE.md) - [Scoring Configuration](./scoring-configuration.md) diff --git a/docs/cli-vs-ui-parity.md b/docs/cli-vs-ui-parity.md index b229e2cac..e74f32689 100644 --- a/docs/cli-vs-ui-parity.md +++ b/docs/cli-vs-ui-parity.md @@ -18,7 +18,7 @@ Status key: | UI capability | CLI command(s) | Status | Notes / Tasks | |---------------|----------------|--------|---------------| | Login / token cache status (`/console/profile`) | `stella auth login`, `stella auth status`, `stella auth whoami` | ✅ Available | Command definitions in `CommandFactory.BuildAuthCommand`. | -| Fresh-auth challenge for sensitive actions | `stella auth fresh-auth` | ✅ Available | Referenced in `docs/15_UI_GUIDE.md` (Admin). | +| Fresh-auth challenge for sensitive actions | `stella auth fresh-auth` | ✅ Available | Referenced in `docs/UI_GUIDE.md` (Admin). | | Tenant switcher (UI shell) | `--tenant` flag across CLI commands | ✅ Available | All multi-tenant commands require explicit `--tenant`. | | Tenant creation / suspension | *(pending CLI)* | 🟩 Planned | No `stella auth tenant *` commands yet – track via `CLI-TEN-47-001` (scopes & tenancy). | @@ -142,7 +142,7 @@ The script should emit a parity report that feeds into the Downloads workspace ( ## 11 · References -- `docs/15_UI_GUIDE.md` – console workflow overview for parity context. +- `docs/UI_GUIDE.md` – console workflow overview for parity context. - `/docs/operations/console-docker-install.md` – CLI parity section for deployments. - `/docs/observability/ui-telemetry.md` – telemetry metrics referencing CLI checks. - `/docs/security/console-security.md` – security metrics & CLI parity expectations. diff --git a/docs/cli/admin-reference.md b/docs/cli/admin-reference.md index 91c225aaf..be2e88a79 100644 --- a/docs/cli/admin-reference.md +++ b/docs/cli/admin-reference.md @@ -455,6 +455,6 @@ stella admin system status ## See Also -- [CLI Reference](../09_API_CLI_REFERENCE.md) -- [Authority Documentation](../11_AUTHORITY.md) +- [CLI Reference](../API_CLI_REFERENCE.md) +- [Authority Documentation](../AUTHORITY.md) - [Operational Procedures](../operations/administration.md) diff --git a/docs/cli/keyboard-shortcuts.md b/docs/cli/keyboard-shortcuts.md index f1dbea3e3..ef680c17c 100644 --- a/docs/cli/keyboard-shortcuts.md +++ b/docs/cli/keyboard-shortcuts.md @@ -229,5 +229,5 @@ When `prefers-reduced-motion` is set: ## Related Documentation - [Triage CLI Reference](./triage-cli.md) -- [Web UI Guide](../15_UI_GUIDE.md) +- [Web UI Guide](../UI_GUIDE.md) - [Accessibility Guide](../accessibility.md) diff --git a/docs/cli/migration-guide.md b/docs/cli/migration-guide.md index c34cc4255..8192bb257 100644 --- a/docs/cli/migration-guide.md +++ b/docs/cli/migration-guide.md @@ -214,6 +214,6 @@ The unified CLI respects all existing environment variables: ## Related Documentation -- [CLI Reference](../09_API_CLI_REFERENCE.md) +- [CLI Reference](../API_CLI_REFERENCE.md) - [Audit Pack Commands](./audit-pack-commands.md) - [Unknowns CLI Reference](./unknowns-cli-reference.md) diff --git a/docs/console/admin-tenants.md b/docs/console/admin-tenants.md index bb0f17b8f..c60ac6d4d 100644 --- a/docs/console/admin-tenants.md +++ b/docs/console/admin-tenants.md @@ -39,5 +39,5 @@ See: ## References -- Console operator guide: `docs/15_UI_GUIDE.md` -- Offline Kit: `docs/24_OFFLINE_KIT.md` +- Console operator guide: `docs/UI_GUIDE.md` +- Offline Kit: `docs/OFFLINE_KIT.md` diff --git a/docs/console/airgap.md b/docs/console/airgap.md index fee7f72dd..85c430fab 100644 --- a/docs/console/airgap.md +++ b/docs/console/airgap.md @@ -48,5 +48,5 @@ Operators need a quick view of: ## References -- Offline Kit packaging and verification: `docs/24_OFFLINE_KIT.md` +- Offline Kit packaging and verification: `docs/OFFLINE_KIT.md` - Air-gap workflows: `docs/airgap/` diff --git a/docs/console/attestor-ui.md b/docs/console/attestor-ui.md index ce1ee3b3a..7b23f6b9b 100644 --- a/docs/console/attestor-ui.md +++ b/docs/console/attestor-ui.md @@ -21,5 +21,5 @@ The Console includes surfaces for viewing and verifying attestations produced by ## References -- Console operator guide: `docs/15_UI_GUIDE.md` -- Offline Kit verification: `docs/24_OFFLINE_KIT.md` +- Console operator guide: `docs/UI_GUIDE.md` +- Offline Kit verification: `docs/OFFLINE_KIT.md` diff --git a/docs/console/forensics.md b/docs/console/forensics.md index 01c2d1524..234f61f7d 100644 --- a/docs/console/forensics.md +++ b/docs/console/forensics.md @@ -35,6 +35,6 @@ Exports are the bridge between online and offline review: ## References -- Console operator guide: `docs/15_UI_GUIDE.md` -- Offline Kit: `docs/24_OFFLINE_KIT.md` -- Vulnerability Explorer guide (triage model): `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` +- Console operator guide: `docs/UI_GUIDE.md` +- Offline Kit: `docs/OFFLINE_KIT.md` +- Vulnerability Explorer guide (triage model): `docs/VULNERABILITY_EXPLORER_GUIDE.md` diff --git a/docs/console/risk-ui.md b/docs/console/risk-ui.md index 4dcf4feb6..5893d205a 100644 --- a/docs/console/risk-ui.md +++ b/docs/console/risk-ui.md @@ -17,4 +17,4 @@ This document describes how risk and explainability concepts should surface in t - Risk model overview: `docs/risk/overview.md` - Policy explainability: `docs/risk/explainability.md` -- Vulnerability Explorer guide: `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` +- Vulnerability Explorer guide: `docs/VULNERABILITY_EXPLORER_GUIDE.md` diff --git a/docs/db/MIGRATION_STRATEGY.md b/docs/db/MIGRATION_STRATEGY.md index 25558c817..6376646ec 100644 --- a/docs/db/MIGRATION_STRATEGY.md +++ b/docs/db/MIGRATION_STRATEGY.md @@ -542,4 +542,4 @@ services.AddHealthChecks() - [PostgreSQL Advisory Locks](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS) - [Zero-Downtime Migrations](https://docs.stellaops.org/operations/migrations) -- [StellaOps CLI Reference](../09_API_CLI_REFERENCE.md) +- [StellaOps CLI Reference](../API_CLI_REFERENCE.md) diff --git a/docs/db/README.md b/docs/db/README.md index b1d40d82e..b5e3fa295 100644 --- a/docs/db/README.md +++ b/docs/db/README.md @@ -65,6 +65,6 @@ Notes: ## Related Documentation -- [Architecture Overview](../07_HIGH_LEVEL_ARCHITECTURE.md) +- [Architecture Overview](../ARCHITECTURE_OVERVIEW.md) - [Module Dossiers](../modules/) -- [Air-Gap Operations](../24_OFFLINE_KIT.md) +- [Air-Gap Operations](../OFFLINE_KIT.md) diff --git a/docs/db/schemas/sync-ledger.md b/docs/db/schemas/sync-ledger.md index 8d8f45b2e..f29cbbc59 100644 --- a/docs/db/schemas/sync-ledger.md +++ b/docs/db/schemas/sync-ledger.md @@ -271,4 +271,4 @@ DROP TABLE IF EXISTS vuln.sync_ledger; - [Concelier Architecture](../../modules/concelier/architecture.md) - [Federation Export Sprint](../../implplan/SPRINT_8200_0014_0002_federation_export.md) - [Federation Import Sprint](../../implplan/SPRINT_8200_0014_0003_federation_import.md) -- [Air-Gap Operation Guide](../../24_OFFLINE_KIT.md) +- [Air-Gap Operation Guide](../../OFFLINE_KIT.md) diff --git a/docs/db/tasks/PHASE_7_CLEANUP.md b/docs/db/tasks/PHASE_7_CLEANUP.md index ad00610fd..2e13b4243 100644 --- a/docs/db/tasks/PHASE_7_CLEANUP.md +++ b/docs/db/tasks/PHASE_7_CLEANUP.md @@ -149,7 +149,7 @@ ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC; Update all documentation to reflect PostgreSQL as the primary database. **Subtasks:** -- [ ] T7.4.1: Update `docs/07_HIGH_LEVEL_ARCHITECTURE.md` +- [ ] T7.4.1: Update `docs/ARCHITECTURE_OVERVIEW.md` - [ ] T7.4.2: Update module architecture docs - [ ] T7.4.3: Update deployment guides - [ ] T7.4.4: Update operations runbooks @@ -180,7 +180,7 @@ Update offline/air-gap kit to include PostgreSQL. - [ ] T7.5.3: Include schema migrations in kit - [ ] T7.5.4: Update kit documentation - [ ] T7.5.5: Test kit installation in air-gapped environment -- [ ] T7.5.6: Update `docs/24_OFFLINE_KIT.md` +- [ ] T7.5.6: Update `docs/OFFLINE_KIT.md` **Air-Gap Kit Structure:** ``` diff --git a/docs/deploy/console.md b/docs/deploy/console.md index 6ec4a877a..2c6fc1089 100644 --- a/docs/deploy/console.md +++ b/docs/deploy/console.md @@ -205,9 +205,9 @@ Troubleshooting steps: - `deploy/helm/stellaops/values-*.yaml` - environment-specific overrides. - `deploy/compose/docker-compose.console.yaml` - Compose bundle. -- `docs/15_UI_GUIDE.md` - Console workflows and offline posture. -- `/docs/security/console-security.md` - CSP and Authority scopes. -- `/docs/24_OFFLINE_KIT.md` - Offline kit packaging and verification. +- `docs/UI_GUIDE.md` - Console workflows and offline posture. +- `/docs/security/console-security.md` - CSP and Authority scopes. +- `/docs/OFFLINE_KIT.md` - Offline kit packaging and verification. - `/docs/modules/devops/runbooks/deployment-runbook.md` (pending) - wider platform deployment steps. --- diff --git a/docs/deployment/VERSION_MATRIX.md b/docs/deployment/VERSION_MATRIX.md index 706c67c4d..c493d3bd0 100644 --- a/docs/deployment/VERSION_MATRIX.md +++ b/docs/deployment/VERSION_MATRIX.md @@ -281,7 +281,7 @@ docker-compose up -d - [Helm Chart Documentation](../deploy/helm/stellaops/README.md) - [Compose Quickstart](../deploy/compose/README.md) -- [Offline Kit Guide](./24_OFFLINE_KIT.md) +- [Offline Kit Guide](./OFFLINE_KIT.md) - [Air-Gap Provenance](../modules/findings-ledger/airgap-provenance.md) - [Staleness Schema](../schemas/ledger-airgap-staleness.schema.json) diff --git a/docs/evaluate/checklist.md b/docs/evaluate/checklist.md index e6745580f..ddd8361b6 100644 --- a/docs/evaluate/checklist.md +++ b/docs/evaluate/checklist.md @@ -8,7 +8,7 @@ ## Day 2–7: Prove Fit -- [ ] Import the [Offline Update Kit](../24_OFFLINE_KIT.md) and confirm feeds refresh with no Internet access. +- [ ] Import the [Offline Update Kit](../OFFLINE_KIT.md) and confirm feeds refresh with no Internet access. - [ ] Apply a sovereign CryptoProfile matching your regulatory environment (FIPS, eIDAS, GOST, SM). - [ ] Run policy simulations with your SBOMs using `stella policy simulate --input `; log explain outcomes for review. - [ ] Validate attestation workflows by exporting DSSE bundles and replaying them on a secondary host. diff --git a/docs/implplan/AGENTS.md b/docs/implplan/AGENTS.md index e8ad981ee..641996303 100644 --- a/docs/implplan/AGENTS.md +++ b/docs/implplan/AGENTS.md @@ -6,7 +6,7 @@ ## Required Reading (treat as read before DOING) - `docs/README.md` -- `docs/07_HIGH_LEVEL_ARCHITECTURE.md` +- `docs/ARCHITECTURE_OVERVIEW.md` - `docs/modules/platform/architecture-overview.md` - `docs/implplan` sprint template rules (see Section “Naming & Structure” below) - Any sprint-specific upstream docs linked from the current sprint file (e.g., crypto audit, replay runbooks, module architecture dossiers referenced in Dependencies/Prereqs sections) diff --git a/docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md b/docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md index 51d3c4101..f23d1f80e 100644 --- a/docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md +++ b/docs/implplan/SPRINT_20251229_049_BE_csproj_audit_maint_tests.md @@ -10,7 +10,7 @@ - Parallel execution is safe across modules with per-project ownership. ## Documentation Prerequisites - docs/README.md -- docs/07_HIGH_LEVEL_ARCHITECTURE.md +- docs/ARCHITECTURE_OVERVIEW.md - docs/modules/platform/architecture-overview.md - Module dossier for each project under review (docs/modules//architecture.md). ## Delivery Tracker diff --git a/docs/implplan/SPRINT_20260104_001_BE_determinism_timeprovider_injection.md b/docs/implplan/SPRINT_20260104_001_BE_determinism_timeprovider_injection.md index bbe9469ad..cbfa8546c 100644 --- a/docs/implplan/SPRINT_20260104_001_BE_determinism_timeprovider_injection.md +++ b/docs/implplan/SPRINT_20260104_001_BE_determinism_timeprovider_injection.md @@ -14,7 +14,7 @@ ## Documentation Prerequisites - docs/README.md -- docs/07_HIGH_LEVEL_ARCHITECTURE.md +- docs/ARCHITECTURE_OVERVIEW.md - AGENTS.md § 8.2 (Deterministic Time & ID Generation) - Module dossier for each project under refactoring. diff --git a/docs/implplan/SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md b/docs/implplan/SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md new file mode 100644 index 000000000..ae43e0f90 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md @@ -0,0 +1,541 @@ +# Sprint 20260105_001_001_BINDEX - Semantic Diffing Phase 1: IR-Level Semantic Analysis + +## Topic & Scope + +Enhance the BinaryIndex module to leverage B2R2's Intermediate Representation (IR) for semantic-level function comparison, moving beyond instruction-byte normalization to true semantic matching that is resilient to compiler optimizations, instruction reordering, and register allocation differences. + +**Advisory Reference:** Product advisory on semantic diffing breakthrough capabilities (Jan 2026) + +**Key Insight:** Current implementation normalizes instruction bytes and computes CFG hashes, but does not lift to B2R2's LowUIR/SSA form for semantic analysis. This limits accuracy on optimized/obfuscated binaries by ~15-20%. + +**Working directory:** `src/BinaryIndex/` + +**Evidence:** New `StellaOps.BinaryIndex.Semantic` library, updated fingerprint generators, integration tests. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| B2R2 v0.9.1+ | Package | Available | +| StellaOps.BinaryIndex.Disassembly | Internal | Stable | +| StellaOps.BinaryIndex.Fingerprints | Internal | Stable | +| StellaOps.BinaryIndex.DeltaSig | Internal | Stable | + +**Parallel Execution:** Tasks SEMD-001 through SEMD-004 can proceed in parallel. SEMD-005+ depend on foundation work. + +--- + +## Documentation Prerequisites + +- `docs/modules/binary-index/architecture.md` +- `docs/modules/binary-index/README.md` +- B2R2 documentation: https://b2r2.org/ +- SemDiff paper: https://arxiv.org/abs/2308.01463 + +--- + +## Problem Analysis + +### Current State + +``` +Binary Input + | + v +B2R2 Disassembly --> Raw Instructions + | + v +Normalization Pipeline --> Normalized Bytes (position-independent) + | + v +Hash Generation --> BasicBlockHash, CfgHash, StringRefsHash + | + v +Fingerprint Matching --> Similarity Score +``` + +**Limitations:** +1. **Instruction-level comparison** - Sensitive to register allocation changes +2. **No semantic lifting** - Cannot detect equivalent operations with different instructions +3. **Optimization blindness** - Loop unrolling, inlining, constant propagation break matches +4. **Basic CFG hashing** - Edge counts/hashes miss semantic equivalence + +### Target State + +``` +Binary Input + | + v +B2R2 Disassembly --> Raw Instructions + | + v +B2R2 IR Lifting --> LowUIR Statements + | + v +SSA Transformation --> SSA Form (optional) + | + v +Semantic Graph Extraction --> Key-Semantics Graph (KSG) + | + v +Graph Fingerprinting --> Semantic Fingerprint + | + v +Graph Isomorphism Check --> Semantic Similarity Score +``` + +--- + +## Architecture Design + +### New Components + +#### 1. IR Lifting Service + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/IrLiftingService.cs +namespace StellaOps.BinaryIndex.Semantic; + +public interface IIrLiftingService +{ + /// + /// Lift disassembled instructions to B2R2 LowUIR. + /// + Task LiftToIrAsync( + DisassembledFunction function, + LiftOptions? options = null, + CancellationToken ct = default); + + /// + /// Transform IR to SSA form for dataflow analysis. + /// + Task TransformToSsaAsync( + LiftedFunction lifted, + CancellationToken ct = default); +} + +public sealed record LiftedFunction( + string Name, + ulong Address, + ImmutableArray Statements, + ImmutableArray BasicBlocks, + ControlFlowGraph Cfg); + +public sealed record SsaFunction( + string Name, + ulong Address, + ImmutableArray Statements, + ImmutableArray BasicBlocks, + DefUseChains DefUse); +``` + +#### 2. Semantic Graph Extractor + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticGraphExtractor.cs +namespace StellaOps.BinaryIndex.Semantic; + +public interface ISemanticGraphExtractor +{ + /// + /// Extract key-semantics graph from lifted IR. + /// Captures: data dependencies, control dependencies, memory operations. + /// + Task ExtractGraphAsync( + LiftedFunction function, + GraphExtractionOptions? options = null, + CancellationToken ct = default); +} + +public sealed record KeySemanticsGraph( + string FunctionName, + ImmutableArray Nodes, + ImmutableArray Edges, + GraphProperties Properties); + +public sealed record SemanticNode( + int Id, + SemanticNodeType Type, // Compute, Load, Store, Branch, Call, Return + string Operation, // add, mul, cmp, etc. + ImmutableArray Operands); + +public sealed record SemanticEdge( + int SourceId, + int TargetId, + SemanticEdgeType Type); // DataDep, ControlDep, MemoryDep + +public enum SemanticNodeType { Compute, Load, Store, Branch, Call, Return, Phi } +public enum SemanticEdgeType { DataDependency, ControlDependency, MemoryDependency } +``` + +#### 3. Semantic Fingerprint Generator + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticFingerprintGenerator.cs +namespace StellaOps.BinaryIndex.Semantic; + +public interface ISemanticFingerprintGenerator +{ + /// + /// Generate semantic fingerprint from key-semantics graph. + /// + Task GenerateAsync( + KeySemanticsGraph graph, + SemanticFingerprintOptions? options = null, + CancellationToken ct = default); +} + +public sealed record SemanticFingerprint( + string FunctionName, + byte[] GraphHash, // 32-byte SHA-256 of canonical graph + byte[] OperationHash, // Hash of operation sequence + byte[] DataFlowHash, // Hash of data dependency patterns + int NodeCount, + int EdgeCount, + int CyclomaticComplexity, + ImmutableArray ApiCalls, // External calls (semantic anchors) + SemanticFingerprintAlgorithm Algorithm); + +public enum SemanticFingerprintAlgorithm +{ + KsgV1, // Key-Semantics Graph v1 + WeisfeilerLehman, // WL graph hashing + GraphletCounting // Graphlet-based similarity +} +``` + +#### 4. Semantic Matcher + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/SemanticMatcher.cs +namespace StellaOps.BinaryIndex.Semantic; + +public interface ISemanticMatcher +{ + /// + /// Compute semantic similarity between two functions. + /// + Task MatchAsync( + SemanticFingerprint a, + SemanticFingerprint b, + MatchOptions? options = null, + CancellationToken ct = default); + + /// + /// Find best matches for a function in a corpus. + /// + Task> FindMatchesAsync( + SemanticFingerprint query, + IAsyncEnumerable corpus, + decimal minSimilarity = 0.7m, + int maxResults = 10, + CancellationToken ct = default); +} + +public sealed record SemanticMatchResult( + string FunctionA, + string FunctionB, + decimal OverallSimilarity, + decimal GraphSimilarity, + decimal DataFlowSimilarity, + decimal ApiCallSimilarity, + MatchConfidence Confidence, + ImmutableArray Deltas); // What changed + +public enum MatchConfidence { VeryHigh, High, Medium, Low, VeryLow } + +public sealed record MatchDelta( + DeltaType Type, + string Description, + decimal Impact); + +public enum DeltaType { NodeAdded, NodeRemoved, EdgeAdded, EdgeRemoved, OperationChanged } +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| 1 | SEMD-001 | TODO | - | Guild | Create `StellaOps.BinaryIndex.Semantic` project structure | +| 2 | SEMD-002 | TODO | - | Guild | Define IR model types (IrStatement, IrBasicBlock, IrOperand) | +| 3 | SEMD-003 | TODO | - | Guild | Define semantic graph model types (KeySemanticsGraph, SemanticNode, SemanticEdge) | +| 4 | SEMD-004 | TODO | - | Guild | Define SemanticFingerprint and matching result types | +| 5 | SEMD-005 | TODO | SEMD-001,002 | Guild | Implement B2R2 IR lifting adapter (LowUIR extraction) | +| 6 | SEMD-006 | TODO | SEMD-005 | Guild | Implement SSA transformation (optional dataflow analysis) | +| 7 | SEMD-007 | TODO | SEMD-003,005 | Guild | Implement KeySemanticsGraph extractor from IR | +| 8 | SEMD-008 | TODO | SEMD-004,007 | Guild | Implement graph canonicalization for deterministic hashing | +| 9 | SEMD-009 | TODO | SEMD-008 | Guild | Implement Weisfeiler-Lehman graph hashing | +| 10 | SEMD-010 | TODO | SEMD-009 | Guild | Implement SemanticFingerprintGenerator | +| 11 | SEMD-011 | TODO | SEMD-010 | Guild | Implement SemanticMatcher with weighted similarity | +| 12 | SEMD-012 | TODO | SEMD-011 | Guild | Integrate semantic fingerprints into PatchDiffEngine | +| 13 | SEMD-013 | TODO | SEMD-012 | Guild | Integrate semantic fingerprints into DeltaSignatureGenerator | +| 14 | SEMD-014 | TODO | SEMD-010 | Guild | Unit tests: IR lifting correctness | +| 15 | SEMD-015 | TODO | SEMD-010 | Guild | Unit tests: Graph extraction determinism | +| 16 | SEMD-016 | TODO | SEMD-011 | Guild | Unit tests: Semantic matching accuracy | +| 17 | SEMD-017 | TODO | SEMD-013 | Guild | Integration tests: End-to-end semantic diffing | +| 18 | SEMD-018 | TODO | SEMD-017 | Guild | Golden corpus: Create test binaries with known semantic equivalences | +| 19 | SEMD-019 | TODO | SEMD-018 | Guild | Benchmark: Compare accuracy vs. instruction-level matching | +| 20 | SEMD-020 | TODO | SEMD-019 | Guild | Documentation: Update architecture.md with semantic diffing | + +--- + +## Task Details + +### SEMD-001: Create Project Structure + +Create new library project for semantic analysis: + +``` +src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Semantic/ + StellaOps.BinaryIndex.Semantic.csproj + IrLiftingService.cs + SemanticGraphExtractor.cs + SemanticFingerprintGenerator.cs + SemanticMatcher.cs + Models/ + IrModels.cs + GraphModels.cs + FingerprintModels.cs + MatchModels.cs + Internal/ + B2R2IrAdapter.cs + GraphCanonicalizer.cs + WeisfeilerLehmanHasher.cs +``` + +**Acceptance Criteria:** +- [ ] Project builds successfully +- [ ] References StellaOps.BinaryIndex.Disassembly +- [ ] References B2R2.FrontEnd.BinLifter + +--- + +### SEMD-005: Implement B2R2 IR Lifting Adapter + +Leverage B2R2's BinLifter to lift raw instructions to LowUIR: + +```csharp +internal sealed class B2R2IrAdapter : IIrLiftingService +{ + public async Task LiftToIrAsync( + DisassembledFunction function, + LiftOptions? options = null, + CancellationToken ct = default) + { + var handle = BinHandle.FromBytes( + function.Architecture.ToB2R2Isa(), + function.RawBytes); + + var lifter = LowUIRHelper.init(handle); + var statements = new List(); + + foreach (var instr in function.Instructions) + { + ct.ThrowIfCancellationRequested(); + + var stmts = LowUIRHelper.translateInstr(lifter, instr.Address); + statements.AddRange(ConvertStatements(stmts)); + } + + var cfg = BuildControlFlowGraph(statements, function.StartAddress); + + return new LiftedFunction( + function.Name, + function.StartAddress, + [.. statements], + ExtractBasicBlocks(cfg), + cfg); + } +} +``` + +**Acceptance Criteria:** +- [ ] Successfully lifts x64 instructions to IR +- [ ] Successfully lifts ARM64 instructions to IR +- [ ] CFG is correctly constructed +- [ ] Memory operations are properly modeled + +--- + +### SEMD-007: Implement Key-Semantics Graph Extractor + +Extract semantic graph capturing: +- **Computation nodes**: Arithmetic, logic, comparison operations +- **Memory nodes**: Load/store operations with abstract addresses +- **Control nodes**: Branches, calls, returns +- **Data dependency edges**: Def-use chains +- **Control dependency edges**: Branch->target relationships + +```csharp +internal sealed class KeySemanticsGraphExtractor : ISemanticGraphExtractor +{ + public async Task ExtractGraphAsync( + LiftedFunction function, + GraphExtractionOptions? options = null, + CancellationToken ct = default) + { + var nodes = new List(); + var edges = new List(); + var defMap = new Dictionary(); // Variable -> defining node + var nodeId = 0; + + foreach (var stmt in function.Statements) + { + ct.ThrowIfCancellationRequested(); + + var node = CreateNode(ref nodeId, stmt); + nodes.Add(node); + + // Add data dependency edges + foreach (var use in GetUses(stmt)) + { + if (defMap.TryGetValue(use, out var defNode)) + { + edges.Add(new SemanticEdge(defNode, node.Id, SemanticEdgeType.DataDependency)); + } + } + + // Track definitions + foreach (var def in GetDefs(stmt)) + { + defMap[def] = node.Id; + } + } + + // Add control dependency edges from CFG + AddControlDependencies(function.Cfg, nodes, edges); + + return new KeySemanticsGraph( + function.Name, + [.. nodes], + [.. edges], + ComputeProperties(nodes, edges)); + } +} +``` + +--- + +### SEMD-009: Implement Weisfeiler-Lehman Graph Hashing + +WL hashing provides stable graph fingerprints: + +```csharp +internal sealed class WeisfeilerLehmanHasher +{ + private readonly int _iterations; + + public WeisfeilerLehmanHasher(int iterations = 3) + { + _iterations = iterations; + } + + public byte[] ComputeHash(KeySemanticsGraph graph) + { + // Initialize labels from node types + var labels = graph.Nodes.ToDictionary( + n => n.Id, + n => ComputeNodeLabel(n)); + + // WL iteration + for (var i = 0; i < _iterations; i++) + { + var newLabels = new Dictionary(); + + foreach (var node in graph.Nodes) + { + var neighbors = graph.Edges + .Where(e => e.SourceId == node.Id || e.TargetId == node.Id) + .Select(e => e.SourceId == node.Id ? e.TargetId : e.SourceId) + .OrderBy(id => labels[id]) + .ToList(); + + var multiset = string.Join(",", neighbors.Select(id => labels[id])); + var newLabel = ComputeLabel(labels[node.Id], multiset); + newLabels[node.Id] = newLabel; + } + + labels = newLabels; + } + + // Compute final hash from sorted labels + var sortedLabels = labels.Values.OrderBy(l => l).ToList(); + var combined = string.Join("|", sortedLabels); + return SHA256.HashData(Encoding.UTF8.GetBytes(combined)); + } +} +``` + +--- + +## Testing Strategy + +### Unit Tests + +| Test Class | Coverage | +|------------|----------| +| `IrLiftingServiceTests` | IR lifting correctness per architecture | +| `SemanticGraphExtractorTests` | Graph construction, edge types, node types | +| `GraphCanonicalizerTests` | Deterministic ordering | +| `WeisfeilerLehmanHasherTests` | Hash stability, collision resistance | +| `SemanticMatcherTests` | Similarity scoring accuracy | + +### Integration Tests + +| Test Class | Coverage | +|------------|----------| +| `EndToEndSemanticDiffTests` | Full pipeline from binary to match result | +| `OptimizationResilienceTests` | Same source, different optimization levels | +| `CompilerVariantTests` | Same source, GCC vs Clang | + +### Golden Corpus + +Create test binaries from known C source with variations: +- `test_func_O0.o` - No optimization +- `test_func_O2.o` - Standard optimization +- `test_func_O3.o` - Aggressive optimization +- `test_func_clang.o` - Different compiler + +All should match semantically despite instruction differences. + +--- + +## Success Metrics + +| Metric | Current | Target | +|--------|---------|--------| +| Semantic match accuracy (optimized binaries) | ~65% | 85%+ | +| False positive rate | ~5% | <2% | +| Match latency (per function) | N/A | <50ms | +| Memory per function | N/A | <10MB | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| B2R2 IR coverage may be incomplete for some instructions | Risk | Fallback to instruction-level matching for unsupported operations | +| WL hashing may produce collisions for small functions | Risk | Combine with operation hash and API call hash | +| SSA transformation adds latency | Trade-off | Make SSA optional, use for high-confidence matching only | +| Graph size explosion for large functions | Risk | Limit node count, use sampling for very large functions | + +--- + +## Next Checkpoints + +- 2026-01-10: SEMD-001 through SEMD-004 (project structure, models) complete +- 2026-01-17: SEMD-005 through SEMD-010 (core implementation) complete +- 2026-01-24: SEMD-011 through SEMD-020 (integration, testing, benchmarks) complete diff --git a/docs/implplan/SPRINT_20260105_001_002_BINDEX_semdiff_corpus.md b/docs/implplan/SPRINT_20260105_001_002_BINDEX_semdiff_corpus.md new file mode 100644 index 000000000..9f6a368d7 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_001_002_BINDEX_semdiff_corpus.md @@ -0,0 +1,592 @@ +# Sprint 20260105_001_002_BINDEX - Semantic Diffing Phase 2: Function Behavior Corpus + +## Topic & Scope + +Build a comprehensive function behavior corpus (similar to Ghidra's BSim/FunctionID) containing fingerprints of known library functions across multiple versions and architectures. This enables identification of functions in stripped binaries by matching against a large corpus of pre-indexed function behaviors. + +**Advisory Reference:** Product advisory on semantic diffing - BSim behavioral similarity against large signature sets. + +**Key Insight:** Current delta signatures are CVE-specific. A large pre-built corpus of "known good" function behaviors enables identifying functions like "this is `memcpy` from glibc 2.31" even in stripped binaries, which is critical for accurate vulnerability attribution. + +**Working directory:** `src/BinaryIndex/` + +**Evidence:** New `StellaOps.BinaryIndex.Corpus` library, corpus ingestion pipeline, PostgreSQL corpus schema. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| SPRINT_20260105_001_001 (IR Semantics) | Sprint | Required for semantic fingerprints | +| StellaOps.BinaryIndex.Semantic | Internal | From Phase 1 | +| PostgreSQL | Infrastructure | Available | +| Package mirrors (Debian, Alpine, RHEL) | External | Available | + +**Parallel Execution:** Corpus connector development (CORP-005-007) can proceed in parallel after CORP-004. + +--- + +## Documentation Prerequisites + +- `docs/modules/binary-index/architecture.md` +- Phase 1 sprint: `docs/implplan/SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md` +- Ghidra BSim documentation: https://ghidra.re/ghidra_docs/api/ghidra/features/bsim/BSimServerAPI.html + +--- + +## Problem Analysis + +### Current State + +- Delta signatures are generated on-demand for specific CVEs +- No pre-built corpus of common library functions +- Cannot identify functions by behavior alone (requires symbols or prior CVE signature) +- Stripped binaries fall back to weaker Build-ID/hash matching + +### Target State + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Function Behavior Corpus │ +│ │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Corpus Ingestion Layer │ │ +│ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │ +│ │ │ GlibcCorpus │ │ OpenSSLCorpus│ │ zlibCorpus │ ... │ │ +│ │ │ Connector │ │ Connector │ │ Connector │ │ │ +│ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ v │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Fingerprint Generation │ │ +│ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ │ +│ │ │ Instruction │ │ Semantic │ │ API Call │ │ │ +│ │ │ Fingerprint │ │ Fingerprint │ │ Fingerprint │ │ │ +│ │ └──────────────┘ └──────────────┘ └──────────────┘ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ v │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Corpus Storage (PostgreSQL) │ │ +│ │ │ │ +│ │ corpus.libraries - Known libraries (glibc, openssl, etc.) │ │ +│ │ corpus.library_versions - Version snapshots │ │ +│ │ corpus.functions - Function metadata │ │ +│ │ corpus.fingerprints - Fingerprint index (semantic + instruction) │ │ +│ │ corpus.function_clusters - Similar function groups │ │ +│ │ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +│ │ │ +│ v │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Query Layer │ │ +│ │ │ │ +│ │ ICorpusQueryService.IdentifyFunctionAsync(fingerprint) │ │ +│ │ -> Returns: [{library: "glibc", version: "2.31", name: "memcpy"}] │ │ +│ │ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Architecture Design + +### Database Schema + +```sql +-- Corpus schema for function behavior database +CREATE SCHEMA IF NOT EXISTS corpus; + +-- Known libraries tracked in corpus +CREATE TABLE corpus.libraries ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, -- glibc, openssl, zlib, curl + description TEXT, + homepage_url TEXT, + source_repo TEXT, -- git URL + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Library versions indexed +CREATE TABLE corpus.library_versions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + library_id UUID NOT NULL REFERENCES corpus.libraries(id), + version TEXT NOT NULL, -- 2.31, 1.1.1n, 1.2.13 + release_date DATE, + is_security_release BOOLEAN DEFAULT false, + source_archive_sha256 TEXT, -- Hash of source tarball + indexed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (library_id, version) +); + +-- Architecture variants +CREATE TABLE corpus.build_variants ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + library_version_id UUID NOT NULL REFERENCES corpus.library_versions(id), + architecture TEXT NOT NULL, -- x86_64, aarch64, armv7 + abi TEXT, -- gnu, musl, msvc + compiler TEXT, -- gcc, clang + compiler_version TEXT, + optimization_level TEXT, -- O0, O2, O3, Os + build_id TEXT, -- ELF Build-ID if available + binary_sha256 TEXT NOT NULL, + indexed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (library_version_id, architecture, abi, compiler, optimization_level) +); + +-- Functions in corpus +CREATE TABLE corpus.functions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + build_variant_id UUID NOT NULL REFERENCES corpus.build_variants(id), + name TEXT NOT NULL, -- Function name (may be mangled) + demangled_name TEXT, -- Demangled C++ name + address BIGINT NOT NULL, + size_bytes INTEGER NOT NULL, + is_exported BOOLEAN DEFAULT false, + is_inline BOOLEAN DEFAULT false, + source_file TEXT, -- Source file if debug info + source_line INTEGER, + UNIQUE (build_variant_id, name, address) +); + +-- Function fingerprints (multiple algorithms per function) +CREATE TABLE corpus.fingerprints ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + function_id UUID NOT NULL REFERENCES corpus.functions(id), + algorithm TEXT NOT NULL, -- semantic_ksg, instruction_bb, cfg_wl + fingerprint BYTEA NOT NULL, -- Variable length depending on algorithm + fingerprint_hex TEXT GENERATED ALWAYS AS (encode(fingerprint, 'hex')) STORED, + metadata JSONB, -- Algorithm-specific metadata + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (function_id, algorithm) +); + +-- Index for fast fingerprint lookup +CREATE INDEX idx_fingerprints_algorithm_hex ON corpus.fingerprints(algorithm, fingerprint_hex); +CREATE INDEX idx_fingerprints_bytea ON corpus.fingerprints USING hash (fingerprint); + +-- Function clusters (similar functions across versions) +CREATE TABLE corpus.function_clusters ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + library_id UUID NOT NULL REFERENCES corpus.libraries(id), + canonical_name TEXT NOT NULL, -- e.g., "memcpy" across all versions + description TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (library_id, canonical_name) +); + +-- Cluster membership +CREATE TABLE corpus.cluster_members ( + cluster_id UUID NOT NULL REFERENCES corpus.function_clusters(id), + function_id UUID NOT NULL REFERENCES corpus.functions(id), + similarity_to_centroid DECIMAL(5,4), + PRIMARY KEY (cluster_id, function_id) +); + +-- CVE associations (which functions are affected by which CVEs) +CREATE TABLE corpus.function_cves ( + function_id UUID NOT NULL REFERENCES corpus.functions(id), + cve_id TEXT NOT NULL, + affected_state TEXT NOT NULL, -- vulnerable, fixed, not_affected + patch_commit TEXT, -- Git commit that fixed + confidence DECIMAL(3,2) NOT NULL, + evidence_type TEXT, -- changelog, commit, advisory + PRIMARY KEY (function_id, cve_id) +); + +-- Ingestion job tracking +CREATE TABLE corpus.ingestion_jobs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + library_id UUID NOT NULL REFERENCES corpus.libraries(id), + job_type TEXT NOT NULL, -- full_ingest, incremental, cve_update + status TEXT NOT NULL DEFAULT 'pending', + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + functions_indexed INTEGER, + errors JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +``` + +### Core Interfaces + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Corpus/ICorpusIngestionService.cs +namespace StellaOps.BinaryIndex.Corpus; + +public interface ICorpusIngestionService +{ + /// + /// Ingest all functions from a library binary. + /// + Task IngestLibraryAsync( + LibraryMetadata metadata, + Stream binaryStream, + IngestionOptions? options = null, + CancellationToken ct = default); + + /// + /// Ingest a specific version range. + /// + Task> IngestVersionRangeAsync( + string libraryName, + VersionRange range, + IAsyncEnumerable binaries, + CancellationToken ct = default); +} + +public sealed record LibraryMetadata( + string Name, + string Version, + string Architecture, + string? Abi, + string? Compiler, + string? OptimizationLevel); + +public sealed record IngestionResult( + Guid JobId, + string LibraryName, + string Version, + int FunctionsIndexed, + int FingerprintsGenerated, + ImmutableArray Errors); +``` + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Corpus/ICorpusQueryService.cs +namespace StellaOps.BinaryIndex.Corpus; + +public interface ICorpusQueryService +{ + /// + /// Identify a function by its fingerprint. + /// + Task> IdentifyFunctionAsync( + FunctionFingerprints fingerprints, + IdentifyOptions? options = null, + CancellationToken ct = default); + + /// + /// Get all functions associated with a CVE. + /// + Task> GetFunctionsForCveAsync( + string cveId, + CancellationToken ct = default); + + /// + /// Get function evolution across versions. + /// + Task GetFunctionEvolutionAsync( + string libraryName, + string functionName, + CancellationToken ct = default); +} + +public sealed record FunctionFingerprints( + byte[]? SemanticHash, + byte[]? InstructionHash, + byte[]? CfgHash, + ImmutableArray? ApiCalls); + +public sealed record FunctionMatch( + string LibraryName, + string Version, + string FunctionName, + decimal Similarity, + MatchConfidence Confidence, + string? CveStatus, // null if not CVE-affected + ImmutableArray AffectedCves); + +public sealed record FunctionEvolution( + string LibraryName, + string FunctionName, + ImmutableArray Versions); + +public sealed record VersionSnapshot( + string Version, + int SizeBytes, + string FingerprintHex, + ImmutableArray CveChanges); // CVEs fixed/introduced in this version +``` + +### Library Connectors + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Corpus/Connectors/IGlibcCorpusConnector.cs +namespace StellaOps.BinaryIndex.Corpus.Connectors; + +public interface ILibraryCorpusConnector +{ + string LibraryName { get; } + string[] SupportedArchitectures { get; } + + /// + /// Get available versions from source. + /// + Task> GetAvailableVersionsAsync(CancellationToken ct); + + /// + /// Download and extract library binary for a version. + /// + Task FetchBinaryAsync( + string version, + string architecture, + string? abi = null, + CancellationToken ct = default); +} + +// Implementations: +// - GlibcCorpusConnector (GNU C Library) +// - OpenSslCorpusConnector (OpenSSL/LibreSSL/BoringSSL) +// - ZlibCorpusConnector (zlib/zlib-ng) +// - CurlCorpusConnector (libcurl) +// - SqliteCorpusConnector (SQLite) +// - LibpngCorpusConnector (libpng) +// - LibjpegCorpusConnector (libjpeg-turbo) +// - LibxmlCorpusConnector (libxml2) +// - OpenJpegCorpusConnector (OpenJPEG) +// - ExpatCorpusConnector (Expat XML parser) +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| 1 | CORP-001 | TODO | Phase 1 | Guild | Create `StellaOps.BinaryIndex.Corpus` project structure | +| 2 | CORP-002 | TODO | CORP-001 | Guild | Define corpus model types (LibraryMetadata, FunctionMatch, etc.) | +| 3 | CORP-003 | TODO | CORP-001 | Guild | Create PostgreSQL corpus schema (corpus.* tables) | +| 4 | CORP-004 | TODO | CORP-003 | Guild | Implement PostgreSQL corpus repository | +| 5 | CORP-005 | TODO | CORP-004 | Guild | Implement GlibcCorpusConnector | +| 6 | CORP-006 | TODO | CORP-004 | Guild | Implement OpenSslCorpusConnector | +| 7 | CORP-007 | TODO | CORP-004 | Guild | Implement ZlibCorpusConnector | +| 8 | CORP-008 | TODO | CORP-004 | Guild | Implement CurlCorpusConnector | +| 9 | CORP-009 | TODO | CORP-005-008 | Guild | Implement CorpusIngestionService | +| 10 | CORP-010 | TODO | CORP-009 | Guild | Implement batch fingerprint generation pipeline | +| 11 | CORP-011 | TODO | CORP-010 | Guild | Implement function clustering (group similar functions) | +| 12 | CORP-012 | TODO | CORP-011 | Guild | Implement CorpusQueryService | +| 13 | CORP-013 | TODO | CORP-012 | Guild | Implement CVE-to-function mapping updater | +| 14 | CORP-014 | TODO | CORP-012 | Guild | Integrate corpus queries into BinaryVulnerabilityService | +| 15 | CORP-015 | TODO | CORP-009 | Guild | Initial corpus ingestion: glibc (5 major versions x 3 archs) | +| 16 | CORP-016 | TODO | CORP-015 | Guild | Initial corpus ingestion: OpenSSL (10 versions x 3 archs) | +| 17 | CORP-017 | TODO | CORP-016 | Guild | Initial corpus ingestion: zlib, curl, sqlite | +| 18 | CORP-018 | TODO | CORP-012 | Guild | Unit tests: Corpus ingestion correctness | +| 19 | CORP-019 | TODO | CORP-012 | Guild | Unit tests: Query service accuracy | +| 20 | CORP-020 | TODO | CORP-017 | Guild | Integration tests: End-to-end function identification | +| 21 | CORP-021 | TODO | CORP-020 | Guild | Benchmark: Query latency at scale (100K+ functions) | +| 22 | CORP-022 | TODO | CORP-021 | Guild | Documentation: Corpus management guide | + +--- + +## Task Details + +### CORP-005: Implement GlibcCorpusConnector + +Fetch glibc binaries from GNU mirrors and Debian/Ubuntu packages: + +```csharp +internal sealed class GlibcCorpusConnector : ILibraryCorpusConnector +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public string LibraryName => "glibc"; + public string[] SupportedArchitectures => ["x86_64", "aarch64", "armv7", "i686"]; + + public async Task> GetAvailableVersionsAsync(CancellationToken ct) + { + // Query GNU FTP mirror for available versions + // https://ftp.gnu.org/gnu/glibc/ + var client = _httpClientFactory.CreateClient("GnuMirror"); + var html = await client.GetStringAsync("https://ftp.gnu.org/gnu/glibc/", ct); + + // Parse directory listing for glibc-X.Y.tar.gz files + var versions = ParseVersionsFromListing(html); + + return [.. versions.OrderByDescending(v => Version.Parse(v))]; + } + + public async Task FetchBinaryAsync( + string version, + string architecture, + string? abi = null, + CancellationToken ct = default) + { + // Strategy 1: Try Debian/Ubuntu package (pre-built) + var debBinary = await TryFetchDebianPackageAsync(version, architecture, ct); + if (debBinary is not null) + return debBinary; + + // Strategy 2: Download source and compile with specific flags + var sourceTarball = await DownloadSourceAsync(version, ct); + return await CompileForArchitecture(sourceTarball, architecture, abi, ct); + } + + private async Task TryFetchDebianPackageAsync( + string version, + string architecture, + CancellationToken ct) + { + // Map glibc version to Debian package version + // e.g., glibc 2.31 -> libc6_2.31-13+deb11u5_amd64.deb + var packages = await QueryDebianPackagesAsync(version, architecture, ct); + + foreach (var pkg in packages) + { + var binary = await DownloadAndExtractDebAsync(pkg, ct); + if (binary is not null) + return binary; + } + + return null; + } +} +``` + +### CORP-011: Implement Function Clustering + +Group semantically similar functions across versions: + +```csharp +internal sealed class FunctionClusteringService +{ + private readonly ICorpusRepository _repository; + private readonly ISemanticMatcher _matcher; + + public async Task ClusterFunctionsAsync( + Guid libraryId, + ClusteringOptions options, + CancellationToken ct) + { + // Get all functions with semantic fingerprints + var functions = await _repository.GetFunctionsWithFingerprintsAsync(libraryId, ct); + + // Group by canonical name (demangled, normalized) + var groups = functions + .GroupBy(f => NormalizeCanonicalName(f.DemangledName ?? f.Name)) + .ToList(); + + foreach (var group in groups) + { + ct.ThrowIfCancellationRequested(); + + // Create or update cluster + var clusterId = await _repository.EnsureClusterAsync( + libraryId, + group.Key, + ct); + + // Compute centroid (most common fingerprint) + var centroid = ComputeCentroid(group); + + // Add members with similarity scores + foreach (var function in group) + { + var similarity = await _matcher.MatchAsync( + function.SemanticFingerprint, + centroid, + ct: ct); + + await _repository.AddClusterMemberAsync( + clusterId, + function.Id, + similarity.OverallSimilarity, + ct); + } + } + } + + private static string NormalizeCanonicalName(string name) + { + // Strip version suffixes, GLIBC_2.X annotations + // Demangle C++ names + // Normalize to base function name + return CppDemangler.Demangle(name) + .Replace("@GLIBC_", "") + .TrimEnd("@@".ToCharArray()); + } +} +``` + +--- + +## Initial Corpus Coverage + +### Priority Libraries (Phase 2a) + +| Library | Versions | Architectures | Est. Functions | CVE Coverage | +|---------|----------|---------------|----------------|--------------| +| glibc | 2.17, 2.28, 2.31, 2.35, 2.38 | x64, arm64, armv7 | ~15,000 | 50+ CVEs | +| OpenSSL | 1.0.2, 1.1.0, 1.1.1, 3.0, 3.1 | x64, arm64 | ~8,000 | 100+ CVEs | +| zlib | 1.2.8, 1.2.11, 1.2.13, 1.3 | x64, arm64 | ~200 | 5+ CVEs | +| libcurl | 7.50-7.88 (select) | x64, arm64 | ~2,000 | 80+ CVEs | +| SQLite | 3.30-3.44 (select) | x64, arm64 | ~1,500 | 30+ CVEs | + +### Extended Coverage (Phase 2b) + +| Library | Est. Functions | Priority | +|---------|----------------|----------| +| libpng | ~300 | Medium | +| libjpeg-turbo | ~400 | Medium | +| libxml2 | ~1,200 | High | +| expat | ~150 | High | +| OpenJPEG | ~600 | Medium | +| freetype | ~800 | Medium | +| harfbuzz | ~500 | Low | + +**Total estimated corpus size:** ~30,000 unique functions, ~100,000 fingerprints (including variants) + +--- + +## Storage Estimates + +| Component | Size Estimate | +|-----------|---------------| +| PostgreSQL tables | ~2 GB | +| Fingerprint index | ~500 MB | +| Full corpus with metadata | ~5 GB | +| Query cache (Valkey) | ~100 MB | + +--- + +## Success Metrics + +| Metric | Target | +|--------|--------| +| Function identification accuracy | 90%+ on stripped binaries | +| Query latency (p99) | <100ms | +| Corpus coverage (top 20 libs) | 80%+ of security-critical functions | +| CVE attribution accuracy | 95%+ | +| False positive rate | <3% | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| Corpus size may grow large | Risk | Implement tiered storage, archive old versions | +| Package version mapping is complex | Risk | Maintain distro-version mapping tables | +| Compilation variants create explosion | Risk | Prioritize common optimization levels (O2, O3) | +| CVE mapping requires manual curation | Risk | Start with high-impact CVEs, automate with NVD data | + +--- + +## Next Checkpoints + +- 2026-01-20: CORP-001 through CORP-008 (infrastructure, connectors) complete +- 2026-01-31: CORP-009 through CORP-014 (services, integration) complete +- 2026-02-15: CORP-015 through CORP-022 (corpus ingestion, testing) complete diff --git a/docs/implplan/SPRINT_20260105_001_003_BINDEX_semdiff_ghidra.md b/docs/implplan/SPRINT_20260105_001_003_BINDEX_semdiff_ghidra.md new file mode 100644 index 000000000..3977a26b5 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_001_003_BINDEX_semdiff_ghidra.md @@ -0,0 +1,772 @@ +# Sprint 20260105_001_003_BINDEX - Semantic Diffing Phase 3: Ghidra Integration + +## Topic & Scope + +Integrate Ghidra as a secondary analysis backend for cases where B2R2 provides insufficient coverage or accuracy. Leverage Ghidra's mature Version Tracking, BSim, and FunctionID capabilities via headless analysis and the ghidriff Python bridge. + +**Advisory Reference:** Product advisory on semantic diffing - Ghidra Version Tracking correlators, BSim behavioral similarity, ghidriff for automated patch diff workflows. + +**Key Insight:** Ghidra has 15+ years of refinement in binary diffing. Rather than reimplementing, we should integrate Ghidra as a fallback/enhancement layer for: +1. Architectures B2R2 handles poorly +2. Complex obfuscation scenarios +3. Version Tracking with multiple correlators +4. BSim database queries + +**Working directory:** `src/BinaryIndex/` + +**Evidence:** New `StellaOps.BinaryIndex.Ghidra` library, Ghidra Headless integration, ghidriff bridge. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| SPRINT_20260105_001_001 (IR Semantics) | Sprint | Should be complete | +| SPRINT_20260105_001_002 (Corpus) | Sprint | Can run in parallel | +| Ghidra 11.x | External | Available | +| Java 17+ | Runtime | Required for Ghidra | +| Python 3.10+ | Runtime | Required for ghidriff | +| ghidriff | External | Available (pip) | + +**Parallel Execution:** Ghidra Headless setup (GHID-001-004) and ghidriff integration (GHID-005-008) can proceed in parallel. + +--- + +## Documentation Prerequisites + +- `docs/modules/binary-index/architecture.md` +- Ghidra documentation: https://ghidra.re/ghidra_docs/ +- Ghidra Version Tracking: https://cve-north-stars.github.io/docs/Ghidra-Patch-Diffing +- ghidriff repository: https://github.com/clearbluejar/ghidriff +- BSim documentation: https://ghidra.re/ghidra_docs/api/ghidra/features/bsim/ + +--- + +## Problem Analysis + +### Current State + +- B2R2 is the sole disassembly/analysis backend +- B2R2 coverage varies by architecture (excellent x64/ARM64, limited others) +- No access to Ghidra's mature correlators and similarity engines +- Cannot leverage BSim's pre-built signature databases + +### B2R2 vs Ghidra Trade-offs + +| Capability | B2R2 | Ghidra | +|------------|------|--------| +| Speed | Fast (native .NET) | Slower (Java, headless startup) | +| Architecture coverage | 12+ (some limited) | 20+ (mature) | +| IR quality | Good (LowUIR) | Excellent (P-Code) | +| Decompiler | None | Excellent | +| Version Tracking | None | Mature (multiple correlators) | +| BSim | None | Full support | +| Integration | Native .NET | Process/API bridge | + +### Target Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Unified Disassembly/Analysis Layer │ +│ │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ IDisassemblyPlugin Selection Logic │ │ +│ │ │ │ +│ │ Primary: B2R2 (fast, deterministic) │ │ +│ │ Fallback: Ghidra (complex cases, low B2R2 confidence) │ │ +│ │ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +│ │ │ │ +│ v v │ +│ ┌──────────────────────────┐ ┌──────────────────────────────────────┐ │ +│ │ B2R2 Backend │ │ Ghidra Backend │ │ +│ │ │ │ │ │ +│ │ - Native .NET │ │ ┌────────────────────────────────┐ │ │ +│ │ - LowUIR lifting │ │ │ Ghidra Headless Server │ │ │ +│ │ - CFG recovery │ │ │ │ │ │ +│ │ - Fast fingerprinting │ │ │ - P-Code decompilation │ │ │ +│ │ │ │ │ - Version Tracking │ │ │ +│ └──────────────────────────┘ │ │ - BSim queries │ │ │ +│ │ │ - FunctionID matching │ │ │ +│ │ └────────────────────────────────┘ │ │ +│ │ │ │ │ +│ │ v │ │ +│ │ ┌────────────────────────────────┐ │ │ +│ │ │ ghidriff Bridge │ │ │ +│ │ │ │ │ │ +│ │ │ - Automated patch diffing │ │ │ +│ │ │ - JSON/Markdown output │ │ │ +│ │ │ - CI/CD integration │ │ │ +│ │ └────────────────────────────────┘ │ │ +│ └──────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Architecture Design + +### Ghidra Headless Service + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ghidra/IGhidraService.cs +namespace StellaOps.BinaryIndex.Ghidra; + +public interface IGhidraService +{ + /// + /// Analyze a binary using Ghidra headless. + /// + Task AnalyzeAsync( + Stream binaryStream, + GhidraAnalysisOptions? options = null, + CancellationToken ct = default); + + /// + /// Run Version Tracking between two binaries. + /// + Task CompareVersionsAsync( + Stream oldBinary, + Stream newBinary, + VersionTrackingOptions? options = null, + CancellationToken ct = default); + + /// + /// Query BSim for function matches. + /// + Task> QueryBSimAsync( + GhidraFunction function, + BSimQueryOptions? options = null, + CancellationToken ct = default); + + /// + /// Check if Ghidra backend is available and healthy. + /// + Task IsAvailableAsync(CancellationToken ct = default); +} + +public sealed record GhidraAnalysisResult( + string BinaryHash, + ImmutableArray Functions, + ImmutableArray Imports, + ImmutableArray Exports, + ImmutableArray Strings, + GhidraMetadata Metadata); + +public sealed record GhidraFunction( + string Name, + ulong Address, + int Size, + string? Signature, // Decompiled signature + string? DecompiledCode, // Decompiled C code + byte[] PCodeHash, // P-Code semantic hash + ImmutableArray CalledFunctions, + ImmutableArray CallingFunctions); +``` + +### Version Tracking Integration + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ghidra/IVersionTrackingService.cs +namespace StellaOps.BinaryIndex.Ghidra; + +public interface IVersionTrackingService +{ + /// + /// Run Ghidra Version Tracking with multiple correlators. + /// + Task TrackVersionsAsync( + Stream oldBinary, + Stream newBinary, + VersionTrackingOptions options, + CancellationToken ct = default); +} + +public sealed record VersionTrackingOptions +{ + public ImmutableArray Correlators { get; init; } = + [CorrelatorType.ExactBytes, CorrelatorType.ExactMnemonics, + CorrelatorType.SymbolName, CorrelatorType.DataReference, + CorrelatorType.CombinedReference]; + + public decimal MinSimilarity { get; init; } = 0.5m; + public bool IncludeDecompilation { get; init; } = false; +} + +public enum CorrelatorType +{ + ExactBytes, // Identical byte sequences + ExactMnemonics, // Identical instruction mnemonics + SymbolName, // Matching symbol names + DataReference, // Similar data references + CombinedReference, // Combined reference scoring + BSim // Behavioral similarity +} + +public sealed record VersionTrackingResult( + ImmutableArray Matches, + ImmutableArray AddedFunctions, + ImmutableArray RemovedFunctions, + ImmutableArray ModifiedFunctions, + VersionTrackingStats Statistics); + +public sealed record FunctionMatch( + string OldName, + ulong OldAddress, + string NewName, + ulong NewAddress, + decimal Similarity, + CorrelatorType MatchedBy, + ImmutableArray Differences); + +public sealed record MatchDifference( + DifferenceType Type, + string Description, + string? OldValue, + string? NewValue); + +public enum DifferenceType +{ + InstructionAdded, + InstructionRemoved, + InstructionChanged, + BranchTargetChanged, + CallTargetChanged, + ConstantChanged, + SizeChanged +} +``` + +### ghidriff Bridge + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ghidra/IGhidriffBridge.cs +namespace StellaOps.BinaryIndex.Ghidra; + +public interface IGhidriffBridge +{ + /// + /// Run ghidriff to compare two binaries. + /// + Task DiffAsync( + string oldBinaryPath, + string newBinaryPath, + GhidriffOptions? options = null, + CancellationToken ct = default); + + /// + /// Generate patch diff report. + /// + Task GenerateReportAsync( + GhidriffResult result, + ReportFormat format, + CancellationToken ct = default); +} + +public sealed record GhidriffOptions +{ + public string? GhidraPath { get; init; } + public string? ProjectPath { get; init; } + public bool IncludeDecompilation { get; init; } = true; + public bool IncludeDisassembly { get; init; } = true; + public ImmutableArray ExcludeFunctions { get; init; } = []; +} + +public sealed record GhidriffResult( + string OldBinaryHash, + string NewBinaryHash, + ImmutableArray AddedFunctions, + ImmutableArray RemovedFunctions, + ImmutableArray ModifiedFunctions, + GhidriffStats Statistics, + string RawJsonOutput); + +public sealed record GhidriffDiff( + string FunctionName, + string OldSignature, + string NewSignature, + decimal Similarity, + string? OldDecompiled, + string? NewDecompiled, + ImmutableArray InstructionChanges); + +public enum ReportFormat { Json, Markdown, Html } +``` + +### BSim Integration + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ghidra/IBSimService.cs +namespace StellaOps.BinaryIndex.Ghidra; + +public interface IBSimService +{ + /// + /// Generate BSim signatures for functions. + /// + Task> GenerateSignaturesAsync( + GhidraAnalysisResult analysis, + BSimGenerationOptions? options = null, + CancellationToken ct = default); + + /// + /// Query BSim database for similar functions. + /// + Task> QueryAsync( + BSimSignature signature, + BSimQueryOptions? options = null, + CancellationToken ct = default); + + /// + /// Ingest functions into BSim database. + /// + Task IngestAsync( + string libraryName, + string version, + ImmutableArray signatures, + CancellationToken ct = default); +} + +public sealed record BSimSignature( + string FunctionName, + ulong Address, + byte[] FeatureVector, // BSim feature extraction + int VectorLength, + double SelfSignificance); // How distinctive is this function + +public sealed record BSimMatch( + string MatchedLibrary, + string MatchedVersion, + string MatchedFunction, + double Similarity, + double Significance, + double Confidence); + +public sealed record BSimQueryOptions +{ + public double MinSimilarity { get; init; } = 0.7; + public double MinSignificance { get; init; } = 0.0; + public int MaxResults { get; init; } = 10; + public ImmutableArray TargetLibraries { get; init; } = []; +} +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| 1 | GHID-001 | TODO | - | Guild | Create `StellaOps.BinaryIndex.Ghidra` project structure | +| 2 | GHID-002 | TODO | GHID-001 | Guild | Define Ghidra model types (GhidraFunction, VersionTrackingResult, etc.) | +| 3 | GHID-003 | TODO | GHID-001 | Guild | Implement Ghidra Headless launcher/manager | +| 4 | GHID-004 | TODO | GHID-003 | Guild | Implement GhidraService (headless analysis wrapper) | +| 5 | GHID-005 | TODO | GHID-001 | Guild | Set up ghidriff Python environment | +| 6 | GHID-006 | TODO | GHID-005 | Guild | Implement GhidriffBridge (Python interop) | +| 7 | GHID-007 | TODO | GHID-006 | Guild | Implement GhidriffReportGenerator | +| 8 | GHID-008 | TODO | GHID-004,006 | Guild | Implement VersionTrackingService | +| 9 | GHID-009 | TODO | GHID-004 | Guild | Implement BSim signature generation | +| 10 | GHID-010 | TODO | GHID-009 | Guild | Implement BSim query service | +| 11 | GHID-011 | TODO | GHID-010 | Guild | Set up BSim PostgreSQL database | +| 12 | GHID-012 | TODO | GHID-008,010 | Guild | Implement GhidraDisassemblyPlugin (IDisassemblyPlugin) | +| 13 | GHID-013 | TODO | GHID-012 | Guild | Integrate Ghidra into DisassemblyService as fallback | +| 14 | GHID-014 | TODO | GHID-013 | Guild | Implement fallback selection logic (B2R2 -> Ghidra) | +| 15 | GHID-015 | TODO | GHID-008 | Guild | Unit tests: Version Tracking correlators | +| 16 | GHID-016 | TODO | GHID-010 | Guild | Unit tests: BSim signature generation | +| 17 | GHID-017 | TODO | GHID-014 | Guild | Integration tests: Fallback scenarios | +| 18 | GHID-018 | TODO | GHID-017 | Guild | Benchmark: Ghidra vs B2R2 accuracy comparison | +| 19 | GHID-019 | TODO | GHID-018 | Guild | Documentation: Ghidra deployment guide | +| 20 | GHID-020 | TODO | GHID-019 | Guild | Docker image: Ghidra Headless service | + +--- + +## Task Details + +### GHID-003: Implement Ghidra Headless Launcher + +Manage Ghidra Headless process lifecycle: + +```csharp +internal sealed class GhidraHeadlessManager : IAsyncDisposable +{ + private readonly GhidraOptions _options; + private readonly ILogger _logger; + private Process? _ghidraProcess; + private readonly SemaphoreSlim _lock = new(1, 1); + + public GhidraHeadlessManager( + IOptions options, + ILogger logger) + { + _options = options.Value; + _logger = logger; + } + + public async Task AnalyzeAsync( + string binaryPath, + string scriptName, + string[] scriptArgs, + CancellationToken ct) + { + await _lock.WaitAsync(ct); + try + { + var projectDir = Path.Combine(_options.WorkDir, Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(projectDir); + + var args = BuildAnalyzeArgs(projectDir, binaryPath, scriptName, scriptArgs); + + var result = await RunGhidraAsync(args, ct); + + return result; + } + finally + { + _lock.Release(); + } + } + + private string[] BuildAnalyzeArgs( + string projectDir, + string binaryPath, + string scriptName, + string[] scriptArgs) + { + var args = new List + { + projectDir, // Project location + "TempProject", // Project name + "-import", binaryPath, + "-postScript", scriptName + }; + + if (scriptArgs.Length > 0) + { + args.AddRange(scriptArgs); + } + + // Add standard options + args.AddRange([ + "-noanalysis", // We'll run analysis explicitly + "-scriptPath", _options.ScriptsDir, + "-max-cpu", _options.MaxCpu.ToString(CultureInfo.InvariantCulture) + ]); + + return [.. args]; + } + + private async Task RunGhidraAsync(string[] args, CancellationToken ct) + { + var startInfo = new ProcessStartInfo + { + FileName = Path.Combine(_options.GhidraHome, "support", "analyzeHeadless"), + Arguments = string.Join(" ", args.Select(QuoteArg)), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + // Set Java options + startInfo.EnvironmentVariables["JAVA_HOME"] = _options.JavaHome; + startInfo.EnvironmentVariables["MAXMEM"] = _options.MaxMemory; + + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Failed to start Ghidra"); + + var output = await process.StandardOutput.ReadToEndAsync(ct); + var error = await process.StandardError.ReadToEndAsync(ct); + + await process.WaitForExitAsync(ct); + + if (process.ExitCode != 0) + { + throw new GhidraException($"Ghidra failed: {error}"); + } + + return output; + } +} +``` + +### GHID-006: Implement ghidriff Bridge + +Python interop for ghidriff: + +```csharp +internal sealed class GhidriffBridge : IGhidriffBridge +{ + private readonly GhidriffOptions _options; + private readonly ILogger _logger; + + public async Task DiffAsync( + string oldBinaryPath, + string newBinaryPath, + GhidriffOptions? options = null, + CancellationToken ct = default) + { + options ??= _options; + + var outputDir = Path.Combine(Path.GetTempPath(), $"ghidriff_{Guid.NewGuid():N}"); + Directory.CreateDirectory(outputDir); + + try + { + var args = BuildGhidriffArgs(oldBinaryPath, newBinaryPath, outputDir, options); + + var result = await RunPythonAsync("ghidriff", args, ct); + + // Parse JSON output + var jsonPath = Path.Combine(outputDir, "diff.json"); + if (!File.Exists(jsonPath)) + { + throw new GhidriffException($"ghidriff did not produce output: {result}"); + } + + var json = await File.ReadAllTextAsync(jsonPath, ct); + return ParseGhidriffOutput(json); + } + finally + { + if (Directory.Exists(outputDir)) + { + Directory.Delete(outputDir, recursive: true); + } + } + } + + private static string[] BuildGhidriffArgs( + string oldPath, + string newPath, + string outputDir, + GhidriffOptions options) + { + var args = new List + { + oldPath, + newPath, + "--output-dir", outputDir, + "--output-format", "json" + }; + + if (!string.IsNullOrEmpty(options.GhidraPath)) + { + args.AddRange(["--ghidra-path", options.GhidraPath]); + } + + if (options.IncludeDecompilation) + { + args.Add("--include-decompilation"); + } + + if (options.ExcludeFunctions.Length > 0) + { + args.AddRange(["--exclude", string.Join(",", options.ExcludeFunctions)]); + } + + return [.. args]; + } + + private async Task RunPythonAsync( + string module, + string[] args, + CancellationToken ct) + { + var startInfo = new ProcessStartInfo + { + FileName = _options.PythonPath ?? "python3", + Arguments = $"-m {module} {string.Join(" ", args.Select(QuoteArg))}", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Failed to start Python"); + + var output = await process.StandardOutput.ReadToEndAsync(ct); + await process.WaitForExitAsync(ct); + + return output; + } +} +``` + +### GHID-014: Implement Fallback Selection Logic + +Smart routing between B2R2 and Ghidra: + +```csharp +internal sealed class HybridDisassemblyService : IDisassemblyService +{ + private readonly B2R2DisassemblyPlugin _b2r2; + private readonly GhidraDisassemblyPlugin _ghidra; + private readonly ILogger _logger; + + public async Task DisassembleAsync( + Stream binaryStream, + DisassemblyOptions? options = null, + CancellationToken ct = default) + { + options ??= new DisassemblyOptions(); + + // Try B2R2 first (faster, native) + var b2r2Result = await TryB2R2Async(binaryStream, options, ct); + + if (b2r2Result is not null && MeetsQualityThreshold(b2r2Result, options)) + { + _logger.LogDebug("Using B2R2 result (confidence: {Confidence})", + b2r2Result.Confidence); + return b2r2Result; + } + + // Fallback to Ghidra for: + // 1. Low B2R2 confidence + // 2. Unsupported architecture + // 3. Explicit Ghidra preference + if (!await _ghidra.IsAvailableAsync(ct)) + { + _logger.LogWarning("Ghidra unavailable, returning B2R2 result"); + return b2r2Result ?? throw new DisassemblyException("No backend available"); + } + + _logger.LogInformation("Falling back to Ghidra (B2R2 confidence: {Confidence})", + b2r2Result?.Confidence ?? 0); + + binaryStream.Position = 0; + return await _ghidra.DisassembleAsync(binaryStream, options, ct); + } + + private static bool MeetsQualityThreshold( + DisassemblyResult result, + DisassemblyOptions options) + { + // Confidence threshold + if (result.Confidence < options.MinConfidence) + return false; + + // Function discovery threshold + if (result.Functions.Length < options.MinFunctions) + return false; + + // Instruction decoding success rate + var decodeRate = (double)result.DecodedInstructions / result.TotalInstructions; + if (decodeRate < options.MinDecodeRate) + return false; + + return true; + } +} +``` + +--- + +## Deployment Architecture + +### Container Setup + +```yaml +# docker-compose.ghidra.yml +services: + ghidra-headless: + image: stellaops/ghidra-headless:11.2 + build: + context: ./devops/docker/ghidra + dockerfile: Dockerfile.headless + volumes: + - ghidra-projects:/projects + - ghidra-scripts:/scripts + environment: + JAVA_HOME: /opt/java/openjdk + MAXMEM: 4G + deploy: + resources: + limits: + cpus: '4' + memory: 8G + + bsim-postgres: + image: postgres:16 + volumes: + - bsim-data:/var/lib/postgresql/data + environment: + POSTGRES_DB: bsim + POSTGRES_USER: bsim + POSTGRES_PASSWORD: ${BSIM_DB_PASSWORD} + +volumes: + ghidra-projects: + ghidra-scripts: + bsim-data: +``` + +### Dockerfile + +```dockerfile +# devops/docker/ghidra/Dockerfile.headless +FROM eclipse-temurin:17-jdk-jammy + +ARG GHIDRA_VERSION=11.2 +ARG GHIDRA_SHA256=abc123... + +# Download and extract Ghidra +RUN curl -fsSL https://github.com/NationalSecurityAgency/ghidra/releases/download/Ghidra_${GHIDRA_VERSION}_build/ghidra_${GHIDRA_VERSION}_PUBLIC_*.zip \ + -o /tmp/ghidra.zip \ + && echo "${GHIDRA_SHA256} /tmp/ghidra.zip" | sha256sum -c - \ + && unzip /tmp/ghidra.zip -d /opt \ + && rm /tmp/ghidra.zip \ + && ln -s /opt/ghidra_* /opt/ghidra + +# Install Python for ghidriff +RUN apt-get update && apt-get install -y python3 python3-pip \ + && pip3 install ghidriff \ + && apt-get clean + +ENV GHIDRA_HOME=/opt/ghidra +ENV PATH="${GHIDRA_HOME}/support:${PATH}" + +WORKDIR /projects +ENTRYPOINT ["analyzeHeadless"] +``` + +--- + +## Success Metrics + +| Metric | Current | Target | +|--------|---------|--------| +| Architecture coverage | 12 (B2R2) | 20+ (with Ghidra) | +| Complex binary accuracy | ~70% | 90%+ | +| Version tracking precision | N/A | 85%+ | +| BSim identification rate | N/A | 80%+ on known libs | +| Fallback latency overhead | N/A | <30s per binary | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| Ghidra adds Java dependency | Trade-off | Containerize Ghidra, keep optional | +| ghidriff Python interop adds complexity | Trade-off | Use subprocess, avoid embedding | +| Ghidra startup time is slow (~10-30s) | Risk | Keep B2R2 primary, Ghidra fallback only | +| BSim database grows large | Risk | Prune old versions, tier storage | +| License considerations (Apache 2.0) | Compliance | Ghidra is Apache 2.0, compatible with AGPL | + +--- + +## Next Checkpoints + +- 2026-02-01: GHID-001 through GHID-007 (project setup, bridges) complete +- 2026-02-15: GHID-008 through GHID-014 (services, integration) complete +- 2026-02-28: GHID-015 through GHID-020 (testing, deployment) complete diff --git a/docs/implplan/SPRINT_20260105_001_004_BINDEX_semdiff_decompiler_ml.md b/docs/implplan/SPRINT_20260105_001_004_BINDEX_semdiff_decompiler_ml.md new file mode 100644 index 000000000..dd3a293bf --- /dev/null +++ b/docs/implplan/SPRINT_20260105_001_004_BINDEX_semdiff_decompiler_ml.md @@ -0,0 +1,906 @@ +# Sprint 20260105_001_004_BINDEX - Semantic Diffing Phase 4: Decompiler Integration & ML Similarity + +## Topic & Scope + +Implement advanced semantic analysis capabilities including decompiled pseudo-code comparison and machine learning-based function embeddings. This phase addresses the highest-impact but most complex enhancements for detecting semantic equivalence in heavily optimized and obfuscated binaries. + +**Advisory Reference:** Product advisory on semantic diffing - SEI Carnegie Mellon semantic equivalence checking of decompiled binaries, ML-based similarity models. + +**Key Insight:** Comparing decompiled C-like code provides the highest semantic fidelity, as it abstracts away instruction-level details. ML embeddings capture functional behavior patterns that resist obfuscation. + +**Working directory:** `src/BinaryIndex/` + +**Evidence:** New `StellaOps.BinaryIndex.Decompiler` and `StellaOps.BinaryIndex.ML` libraries, model training pipeline. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| SPRINT_20260105_001_001 (IR Semantics) | Sprint | Required | +| SPRINT_20260105_001_002 (Corpus) | Sprint | Required for training data | +| SPRINT_20260105_001_003 (Ghidra) | Sprint | Required for decompiler | +| Ghidra Decompiler | External | Via Phase 3 | +| ONNX Runtime | Package | Available | +| ML.NET | Package | Available | + +**Parallel Execution:** Decompiler integration (DCML-001-010) and ML pipeline (DCML-011-020) can proceed in parallel. + +--- + +## Documentation Prerequisites + +- Phase 1-3 sprint documents +- `docs/modules/binary-index/architecture.md` +- SEI paper: https://www.sei.cmu.edu/annual-reviews/2022-research-review/semantic-equivalence-checking-of-decompiled-binaries/ +- Code similarity research: https://arxiv.org/abs/2308.01463 + +--- + +## Problem Analysis + +### Current State + +After Phases 1-3: +- B2R2 IR-level semantic fingerprints (Phase 1) +- Function behavior corpus (Phase 2) +- Ghidra fallback with Version Tracking (Phase 3) + +**Remaining Gaps:** +1. No decompiled code comparison (highest semantic fidelity) +2. No ML-based similarity (robustness to obfuscation) +3. Cannot detect functionally equivalent code with radically different structure + +### Target Capabilities + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Advanced Semantic Analysis Stack │ +│ │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Decompilation Layer │ │ +│ │ │ │ +│ │ Binary -> Ghidra P-Code -> Decompiled C -> AST -> Semantic Hash │ │ +│ │ │ │ +│ │ Comparison methods: │ │ +│ │ - AST structural similarity │ │ +│ │ - Control flow equivalence │ │ +│ │ - Data flow equivalence │ │ +│ │ - Normalized code text similarity │ │ +│ │ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ ML Embedding Layer │ │ +│ │ │ │ +│ │ Function Code -> Tokenization -> Transformer -> Embedding Vector │ │ +│ │ │ │ +│ │ Models: │ │ +│ │ - CodeBERT variant for binary code │ │ +│ │ - Graph Neural Network for CFG │ │ +│ │ - Contrastive learning for similarity │ │ +│ │ │ │ +│ │ Vector similarity: cosine, euclidean, learned metric │ │ +│ │ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌───────────────────────────────────────────────────────────────────────┐ │ +│ │ Ensemble Decision Layer │ │ +│ │ │ │ +│ │ Combine signals: │ │ +│ │ - Instruction fingerprint (Phase 1) : 15% weight │ │ +│ │ - Semantic graph (Phase 1) : 25% weight │ │ +│ │ - Decompiled AST similarity : 35% weight │ │ +│ │ - ML embedding similarity : 25% weight │ │ +│ │ │ │ +│ │ Output: Confidence-weighted similarity score │ │ +│ │ │ │ +│ └───────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Architecture Design + +### Decompiler Integration + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Decompiler/IDecompilerService.cs +namespace StellaOps.BinaryIndex.Decompiler; + +public interface IDecompilerService +{ + /// + /// Decompile a function to C-like pseudo-code. + /// + Task DecompileAsync( + GhidraFunction function, + DecompileOptions? options = null, + CancellationToken ct = default); + + /// + /// Parse decompiled code into AST. + /// + Task ParseToAstAsync( + string decompiledCode, + CancellationToken ct = default); + + /// + /// Compare two decompiled functions for semantic equivalence. + /// + Task CompareAsync( + DecompiledFunction a, + DecompiledFunction b, + ComparisonOptions? options = null, + CancellationToken ct = default); +} + +public sealed record DecompiledFunction( + string FunctionName, + string Signature, + string Code, // Decompiled C code + DecompiledAst? Ast, + ImmutableArray Locals, + ImmutableArray CalledFunctions); + +public sealed record DecompiledAst( + AstNode Root, + int NodeCount, + int Depth, + ImmutableArray Patterns); // Recognized code patterns + +public abstract record AstNode(AstNodeType Type, ImmutableArray Children); + +public enum AstNodeType +{ + Function, Block, If, While, For, DoWhile, Switch, + Return, Break, Continue, Goto, + Assignment, BinaryOp, UnaryOp, Call, Cast, + Variable, Constant, ArrayAccess, FieldAccess, Deref +} +``` + +### AST Comparison Engine + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Decompiler/AstComparisonEngine.cs +namespace StellaOps.BinaryIndex.Decompiler; + +public interface IAstComparisonEngine +{ + /// + /// Compute structural similarity between ASTs. + /// + decimal ComputeStructuralSimilarity(DecompiledAst a, DecompiledAst b); + + /// + /// Compute edit distance between ASTs. + /// + AstEditDistance ComputeEditDistance(DecompiledAst a, DecompiledAst b); + + /// + /// Find semantic equivalent patterns. + /// + ImmutableArray FindEquivalences( + DecompiledAst a, + DecompiledAst b); +} + +public sealed record AstEditDistance( + int Insertions, + int Deletions, + int Modifications, + int TotalOperations, + decimal NormalizedDistance); // 0.0 = identical, 1.0 = completely different + +public sealed record SemanticEquivalence( + AstNode NodeA, + AstNode NodeB, + EquivalenceType Type, + decimal Confidence); + +public enum EquivalenceType +{ + Identical, // Exact match + Renamed, // Same structure, different names + Reordered, // Same operations, different order + Optimized, // Compiler optimization variant + Semantically, // Different structure, same behavior +} +``` + +### Decompiled Code Normalizer + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Decompiler/CodeNormalizer.cs +namespace StellaOps.BinaryIndex.Decompiler; + +public interface ICodeNormalizer +{ + /// + /// Normalize decompiled code for comparison. + /// + string Normalize(string code, NormalizationOptions? options = null); + + /// + /// Generate canonical form hash. + /// + byte[] ComputeCanonicalHash(string code); +} + +internal sealed class CodeNormalizer : ICodeNormalizer +{ + public string Normalize(string code, NormalizationOptions? options = null) + { + options ??= NormalizationOptions.Default; + + var normalized = code; + + // 1. Normalize variable names (var1, var2, ...) + if (options.NormalizeVariables) + { + normalized = NormalizeVariableNames(normalized); + } + + // 2. Normalize function calls (func1, func2, ... or keep known names) + if (options.NormalizeFunctionCalls) + { + normalized = NormalizeFunctionCalls(normalized, options.KnownFunctions); + } + + // 3. Normalize constants (replace magic numbers with placeholders) + if (options.NormalizeConstants) + { + normalized = NormalizeConstants(normalized); + } + + // 4. Normalize whitespace + if (options.NormalizeWhitespace) + { + normalized = NormalizeWhitespace(normalized); + } + + // 5. Sort independent statements (where order doesn't matter) + if (options.SortIndependentStatements) + { + normalized = SortIndependentStatements(normalized); + } + + return normalized; + } + + private static string NormalizeVariableNames(string code) + { + // Replace all local variable names with canonical names + // var_0, var_1, ... in order of first appearance + var varIndex = 0; + var varMap = new Dictionary(); + + // Regex to find variable declarations and uses + return Regex.Replace(code, @"\b([a-zA-Z_][a-zA-Z0-9_]*)\b", match => + { + var name = match.Value; + + // Skip keywords and known types + if (IsKeywordOrType(name)) + return name; + + if (!varMap.TryGetValue(name, out var canonical)) + { + canonical = $"var_{varIndex++}"; + varMap[name] = canonical; + } + + return canonical; + }); + } +} +``` + +### ML Embedding Service + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.ML/IEmbeddingService.cs +namespace StellaOps.BinaryIndex.ML; + +public interface IEmbeddingService +{ + /// + /// Generate embedding vector for a function. + /// + Task GenerateEmbeddingAsync( + EmbeddingInput input, + EmbeddingOptions? options = null, + CancellationToken ct = default); + + /// + /// Compute similarity between embeddings. + /// + decimal ComputeSimilarity( + FunctionEmbedding a, + FunctionEmbedding b, + SimilarityMetric metric = SimilarityMetric.Cosine); + + /// + /// Find similar functions in embedding index. + /// + Task> FindSimilarAsync( + FunctionEmbedding query, + int topK = 10, + decimal minSimilarity = 0.7m, + CancellationToken ct = default); +} + +public sealed record EmbeddingInput( + string? DecompiledCode, // Preferred + KeySemanticsGraph? SemanticGraph, // Fallback + byte[]? InstructionBytes, // Last resort + EmbeddingInputType PreferredInput); + +public enum EmbeddingInputType { DecompiledCode, SemanticGraph, Instructions } + +public sealed record FunctionEmbedding( + string FunctionName, + float[] Vector, // 768-dimensional + EmbeddingModel Model, + EmbeddingInputType InputType); + +public enum EmbeddingModel +{ + CodeBertBinary, // Fine-tuned CodeBERT for binary code + GraphSageFunction, // GNN for CFG/call graph + ContrastiveFunction // Contrastive learning model +} + +public enum SimilarityMetric { Cosine, Euclidean, Manhattan, LearnedMetric } +``` + +### Model Training Pipeline + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.ML/IModelTrainingService.cs +namespace StellaOps.BinaryIndex.ML; + +public interface IModelTrainingService +{ + /// + /// Train embedding model on function pairs. + /// + Task TrainAsync( + IAsyncEnumerable trainingData, + TrainingOptions options, + IProgress? progress = null, + CancellationToken ct = default); + + /// + /// Evaluate model on test set. + /// + Task EvaluateAsync( + IAsyncEnumerable testData, + CancellationToken ct = default); + + /// + /// Export trained model for inference. + /// + Task ExportModelAsync( + string outputPath, + ModelExportFormat format = ModelExportFormat.Onnx, + CancellationToken ct = default); +} + +public sealed record TrainingPair( + EmbeddingInput FunctionA, + EmbeddingInput FunctionB, + bool IsSimilar, // Ground truth: same function? + decimal? SimilarityScore); // Optional: how similar (0-1) + +public sealed record TrainingOptions +{ + public EmbeddingModel Model { get; init; } = EmbeddingModel.CodeBertBinary; + public int EmbeddingDimension { get; init; } = 768; + public int BatchSize { get; init; } = 32; + public int Epochs { get; init; } = 10; + public double LearningRate { get; init; } = 1e-5; + public double MarginLoss { get; init; } = 0.5; // Contrastive margin + public string? PretrainedModelPath { get; init; } +} + +public sealed record TrainingResult( + string ModelPath, + int TotalPairs, + int Epochs, + double FinalLoss, + double ValidationAccuracy, + TimeSpan TrainingTime); + +public sealed record EvaluationResult( + double Accuracy, + double Precision, + double Recall, + double F1Score, + double AucRoc, + ImmutableArray ConfusionMatrix); +``` + +### ONNX Inference Engine + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.ML/OnnxInferenceEngine.cs +namespace StellaOps.BinaryIndex.ML; + +internal sealed class OnnxInferenceEngine : IEmbeddingService, IAsyncDisposable +{ + private readonly InferenceSession _session; + private readonly ITokenizer _tokenizer; + private readonly ILogger _logger; + + public OnnxInferenceEngine( + string modelPath, + ITokenizer tokenizer, + ILogger logger) + { + var options = new SessionOptions + { + GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL, + ExecutionMode = ExecutionMode.ORT_PARALLEL + }; + + _session = new InferenceSession(modelPath, options); + _tokenizer = tokenizer; + _logger = logger; + } + + public async Task GenerateEmbeddingAsync( + EmbeddingInput input, + EmbeddingOptions? options = null, + CancellationToken ct = default) + { + var text = input.PreferredInput switch + { + EmbeddingInputType.DecompiledCode => input.DecompiledCode + ?? throw new ArgumentException("DecompiledCode required"), + EmbeddingInputType.SemanticGraph => SerializeGraph(input.SemanticGraph + ?? throw new ArgumentException("SemanticGraph required")), + EmbeddingInputType.Instructions => SerializeInstructions(input.InstructionBytes + ?? throw new ArgumentException("InstructionBytes required")), + _ => throw new ArgumentOutOfRangeException() + }; + + // Tokenize + var tokens = _tokenizer.Tokenize(text, maxLength: 512); + + // Run inference + var inputTensor = new DenseTensor(tokens, [1, tokens.Length]); + var inputs = new List + { + NamedOnnxValue.CreateFromTensor("input_ids", inputTensor) + }; + + using var results = await Task.Run(() => _session.Run(inputs), ct); + + var outputTensor = results.First().AsTensor(); + var embedding = outputTensor.ToArray(); + + return new FunctionEmbedding( + input.DecompiledCode?.GetHashCode().ToString() ?? "unknown", + embedding, + EmbeddingModel.CodeBertBinary, + input.PreferredInput); + } + + public decimal ComputeSimilarity( + FunctionEmbedding a, + FunctionEmbedding b, + SimilarityMetric metric = SimilarityMetric.Cosine) + { + return metric switch + { + SimilarityMetric.Cosine => CosineSimilarity(a.Vector, b.Vector), + SimilarityMetric.Euclidean => EuclideanSimilarity(a.Vector, b.Vector), + SimilarityMetric.Manhattan => ManhattanSimilarity(a.Vector, b.Vector), + _ => throw new ArgumentOutOfRangeException(nameof(metric)) + }; + } + + private static decimal CosineSimilarity(float[] a, float[] b) + { + var dotProduct = 0.0; + var normA = 0.0; + var normB = 0.0; + + for (var i = 0; i < a.Length; i++) + { + dotProduct += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + + if (normA == 0 || normB == 0) + return 0; + + return (decimal)(dotProduct / (Math.Sqrt(normA) * Math.Sqrt(normB))); + } +} +``` + +### Ensemble Decision Engine + +```csharp +// src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Ensemble/IEnsembleDecisionEngine.cs +namespace StellaOps.BinaryIndex.Ensemble; + +public interface IEnsembleDecisionEngine +{ + /// + /// Compute final similarity using all available signals. + /// + Task ComputeSimilarityAsync( + FunctionAnalysis a, + FunctionAnalysis b, + EnsembleOptions? options = null, + CancellationToken ct = default); +} + +public sealed record FunctionAnalysis( + string FunctionName, + byte[]? InstructionFingerprint, // Phase 1 + SemanticFingerprint? SemanticGraph, // Phase 1 + DecompiledFunction? Decompiled, // Phase 4 + FunctionEmbedding? Embedding); // Phase 4 + +public sealed record EnsembleOptions +{ + // Weight configuration (must sum to 1.0) + public decimal InstructionWeight { get; init; } = 0.15m; + public decimal SemanticGraphWeight { get; init; } = 0.25m; + public decimal DecompiledWeight { get; init; } = 0.35m; + public decimal EmbeddingWeight { get; init; } = 0.25m; + + // Confidence thresholds + public decimal MinConfidence { get; init; } = 0.6m; + public bool RequireAllSignals { get; init; } = false; +} + +public sealed record EnsembleResult( + decimal OverallSimilarity, + MatchConfidence Confidence, + ImmutableArray Contributions, + string? Explanation); + +public sealed record SignalContribution( + string SignalName, + decimal RawSimilarity, + decimal Weight, + decimal WeightedContribution, + bool WasAvailable); +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| **Decompiler Integration** | +| 1 | DCML-001 | TODO | Phase 3 | Guild | Create `StellaOps.BinaryIndex.Decompiler` project | +| 2 | DCML-002 | TODO | DCML-001 | Guild | Define decompiled code model types | +| 3 | DCML-003 | TODO | DCML-002 | Guild | Implement Ghidra decompiler adapter | +| 4 | DCML-004 | TODO | DCML-003 | Guild | Implement C code parser (AST generation) | +| 5 | DCML-005 | TODO | DCML-004 | Guild | Implement AST comparison engine | +| 6 | DCML-006 | TODO | DCML-005 | Guild | Implement code normalizer | +| 7 | DCML-007 | TODO | DCML-006 | Guild | Implement semantic equivalence detector | +| 8 | DCML-008 | TODO | DCML-007 | Guild | Unit tests: Decompiler adapter | +| 9 | DCML-009 | TODO | DCML-007 | Guild | Unit tests: AST comparison | +| 10 | DCML-010 | TODO | DCML-009 | Guild | Integration tests: End-to-end decompiled comparison | +| **ML Embedding Pipeline** | +| 11 | DCML-011 | TODO | Phase 2 | Guild | Create `StellaOps.BinaryIndex.ML` project | +| 12 | DCML-012 | TODO | DCML-011 | Guild | Define embedding model types | +| 13 | DCML-013 | TODO | DCML-012 | Guild | Implement code tokenizer (binary-aware BPE) | +| 14 | DCML-014 | TODO | DCML-013 | Guild | Set up ONNX Runtime inference engine | +| 15 | DCML-015 | TODO | DCML-014 | Guild | Implement embedding service | +| 16 | DCML-016 | TODO | DCML-015 | Guild | Create training data from corpus (positive/negative pairs) | +| 17 | DCML-017 | TODO | DCML-016 | Guild | Train CodeBERT-Binary model | +| 18 | DCML-018 | TODO | DCML-017 | Guild | Export model to ONNX format | +| 19 | DCML-019 | TODO | DCML-015 | Guild | Unit tests: Embedding generation | +| 20 | DCML-020 | TODO | DCML-018 | Guild | Evaluation: Model accuracy metrics | +| **Ensemble Integration** | +| 21 | DCML-021 | TODO | DCML-010,020 | Guild | Create `StellaOps.BinaryIndex.Ensemble` project | +| 22 | DCML-022 | TODO | DCML-021 | Guild | Implement ensemble decision engine | +| 23 | DCML-023 | TODO | DCML-022 | Guild | Implement weight tuning (grid search) | +| 24 | DCML-024 | TODO | DCML-023 | Guild | Integrate ensemble into PatchDiffEngine | +| 25 | DCML-025 | TODO | DCML-024 | Guild | Integrate ensemble into DeltaSignatureMatcher | +| 26 | DCML-026 | TODO | DCML-025 | Guild | Unit tests: Ensemble decision logic | +| 27 | DCML-027 | TODO | DCML-026 | Guild | Integration tests: Full semantic diffing pipeline | +| 28 | DCML-028 | TODO | DCML-027 | Guild | Benchmark: Accuracy vs. baseline (Phase 1 only) | +| 29 | DCML-029 | TODO | DCML-028 | Guild | Benchmark: Latency impact | +| 30 | DCML-030 | TODO | DCML-029 | Guild | Documentation: ML model training guide | + +--- + +## Task Details + +### DCML-004: Implement C Code Parser + +Parse Ghidra's decompiled C output into AST: + +```csharp +internal sealed class DecompiledCodeParser +{ + public DecompiledAst Parse(string code) + { + // Use Tree-sitter or Roslyn-based C parser + // Ghidra output is C-like but not standard C + + var tokens = Tokenize(code); + var ast = BuildAst(tokens); + + return new DecompiledAst( + ast, + CountNodes(ast), + ComputeDepth(ast), + ExtractPatterns(ast)); + } + + private AstNode BuildAst(IList tokens) + { + var parser = new RecursiveDescentParser(tokens); + return parser.ParseFunction(); + } + + private ImmutableArray ExtractPatterns(AstNode root) + { + var patterns = new List(); + + // Detect common patterns + patterns.AddRange(DetectLoopPatterns(root)); + patterns.AddRange(DetectBranchPatterns(root)); + patterns.AddRange(DetectAllocationPatterns(root)); + patterns.AddRange(DetectErrorHandlingPatterns(root)); + + return [.. patterns]; + } + + private static IEnumerable DetectLoopPatterns(AstNode root) + { + // Find: for loops, while loops, do-while + // Classify: counted loop, sentinel loop, infinite loop + foreach (var node in TraverseNodes(root)) + { + if (node.Type == AstNodeType.For) + { + yield return new AstPattern( + PatternType.CountedLoop, + node, + AnalyzeForLoop(node)); + } + else if (node.Type == AstNodeType.While) + { + yield return new AstPattern( + PatternType.ConditionalLoop, + node, + AnalyzeWhileLoop(node)); + } + } + } +} +``` + +### DCML-017: Train CodeBERT-Binary Model + +Training pipeline for function similarity: + +```python +# tools/ml/train_codebert_binary.py +import torch +from transformers import RobertaTokenizer, RobertaModel +from torch.utils.data import DataLoader +import onnx + +class CodeBertBinaryModel(torch.nn.Module): + def __init__(self, pretrained_model="microsoft/codebert-base"): + super().__init__() + self.encoder = RobertaModel.from_pretrained(pretrained_model) + self.projection = torch.nn.Linear(768, 768) + + def forward(self, input_ids, attention_mask): + outputs = self.encoder(input_ids, attention_mask=attention_mask) + pooled = outputs.last_hidden_state[:, 0, :] # [CLS] token + projected = self.projection(pooled) + return torch.nn.functional.normalize(projected, p=2, dim=1) + + +class ContrastiveLoss(torch.nn.Module): + def __init__(self, margin=0.5): + super().__init__() + self.margin = margin + + def forward(self, embedding_a, embedding_b, label): + distance = torch.nn.functional.pairwise_distance(embedding_a, embedding_b) + + # label=1: similar, label=0: dissimilar + loss = label * distance.pow(2) + \ + (1 - label) * torch.clamp(self.margin - distance, min=0).pow(2) + + return loss.mean() + + +def train_model(train_dataloader, val_dataloader, epochs=10): + model = CodeBertBinaryModel() + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + criterion = ContrastiveLoss(margin=0.5) + + for epoch in range(epochs): + model.train() + total_loss = 0 + + for batch in train_dataloader: + optimizer.zero_grad() + + emb_a = model(batch['input_ids_a'], batch['attention_mask_a']) + emb_b = model(batch['input_ids_b'], batch['attention_mask_b']) + + loss = criterion(emb_a, emb_b, batch['label']) + loss.backward() + optimizer.step() + + total_loss += loss.item() + + # Validation + model.eval() + val_accuracy = evaluate(model, val_dataloader) + print(f"Epoch {epoch+1}: Loss={total_loss:.4f}, Val Acc={val_accuracy:.4f}") + + return model + + +def export_to_onnx(model, output_path): + model.eval() + dummy_input = torch.randint(0, 50000, (1, 512)) + dummy_mask = torch.ones(1, 512) + + torch.onnx.export( + model, + (dummy_input, dummy_mask), + output_path, + input_names=['input_ids', 'attention_mask'], + output_names=['embedding'], + dynamic_axes={ + 'input_ids': {0: 'batch', 1: 'seq'}, + 'attention_mask': {0: 'batch', 1: 'seq'}, + 'embedding': {0: 'batch'} + } + ) +``` + +### DCML-023: Implement Weight Tuning + +Grid search for optimal ensemble weights: + +```csharp +internal sealed class EnsembleWeightTuner +{ + public async Task TuneWeightsAsync( + IAsyncEnumerable validationData, + CancellationToken ct) + { + var bestOptions = EnsembleOptions.Default; + var bestF1 = 0.0; + + // Grid search over weight combinations + var weightCombinations = GenerateWeightCombinations(step: 0.05m); + + foreach (var weights in weightCombinations) + { + ct.ThrowIfCancellationRequested(); + + var options = new EnsembleOptions + { + InstructionWeight = weights[0], + SemanticGraphWeight = weights[1], + DecompiledWeight = weights[2], + EmbeddingWeight = weights[3] + }; + + var metrics = await EvaluateAsync(validationData, options, ct); + + if (metrics.F1Score > bestF1) + { + bestF1 = metrics.F1Score; + bestOptions = options; + } + } + + return bestOptions; + } + + private static IEnumerable GenerateWeightCombinations(decimal step) + { + for (var w1 = 0m; w1 <= 1m; w1 += step) + for (var w2 = 0m; w2 <= 1m - w1; w2 += step) + for (var w3 = 0m; w3 <= 1m - w1 - w2; w3 += step) + { + var w4 = 1m - w1 - w2 - w3; + if (w4 >= 0) + { + yield return [w1, w2, w3, w4]; + } + } + } +} +``` + +--- + +## Training Data Requirements + +### Positive Pairs (Similar Functions) + +| Source | Count | Description | +|--------|-------|-------------| +| Same function, different optimization | ~50,000 | O0 vs O2 vs O3 | +| Same function, different compiler | ~30,000 | GCC vs Clang | +| Same function, different version | ~100,000 | From corpus (Phase 2) | +| Same function, with patches | ~20,000 | Vulnerable vs fixed | + +### Negative Pairs (Dissimilar Functions) + +| Source | Count | Description | +|--------|-------|-------------| +| Random function pairs | ~100,000 | Random sampling | +| Similar-named different functions | ~50,000 | Hard negatives | +| Same library, different functions | ~50,000 | Medium negatives | + +**Total training data:** ~400,000 labeled pairs + +--- + +## Success Metrics + +| Metric | Phase 1 Only | With Phase 4 | Target | +|--------|--------------|--------------|--------| +| Accuracy (optimized binaries) | 70% | 92% | 90%+ | +| Accuracy (obfuscated binaries) | 40% | 75% | 70%+ | +| False positive rate | 5% | 1.5% | <2% | +| False negative rate | 25% | 8% | <10% | +| Latency (per comparison) | 10ms | 150ms | <200ms | + +--- + +## Resource Requirements + +| Resource | Training | Inference | +|----------|----------|-----------| +| GPU | 1x V100 (32GB) or 4x T4 | Optional (CPU viable) | +| Memory | 64GB | 16GB | +| Storage | 100GB (training data) | 5GB (model) | +| Time | ~24 hours | <200ms per function | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| ML model requires significant training data | Risk | Leverage corpus from Phase 2 | +| ONNX inference adds latency | Trade-off | Make ML optional, use for high-value comparisons | +| Decompiler output varies by Ghidra version | Risk | Pin Ghidra version, normalize output | +| Model may overfit to training library set | Risk | Diverse training data, regularization | +| GPU dependency for training | Constraint | Use cloud GPU, document CPU-only option | + +--- + +## Next Checkpoints + +- 2026-03-01: DCML-001 through DCML-010 (decompiler integration) complete +- 2026-03-15: DCML-011 through DCML-020 (ML pipeline) complete +- 2026-03-31: DCML-021 through DCML-030 (ensemble, benchmarks) complete diff --git a/docs/implplan/SPRINT_20260105_002_000_INDEX_hlc_audit_safe_ordering.md b/docs/implplan/SPRINT_20260105_002_000_INDEX_hlc_audit_safe_ordering.md new file mode 100644 index 000000000..1393af7ab --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_000_INDEX_hlc_audit_safe_ordering.md @@ -0,0 +1,203 @@ +# Sprint Series 20260105_002 - HLC: Audit-Safe Job Queue Ordering + +## Executive Summary + +This sprint series implements the "Audit-safe job queue ordering" product advisory, adding Hybrid Logical Clock (HLC) based ordering with cryptographic sequence proofs to the StellaOps Scheduler. This closes the ~30% compliance gap identified in the advisory analysis. + +## Problem Statement + +Current StellaOps architecture relies on: +- Wall-clock timestamps (`TimeProvider.GetUtcNow()`) for job ordering +- Per-module sequence numbers (local ordering, not global) +- Hash chains only in downstream ledgers (Findings, Orchestrator Audit) + +This creates risks in: +- **Distributed deployments** with clock skew between nodes +- **Offline/air-gap scenarios** where jobs enqueued offline must merge deterministically +- **Audit forensics** where "prove job A preceded job B" requires global ordering + +## Solution Architecture + +``` + ┌─────────────────────────────────────────────────┐ + │ HLC Core Library │ + │ (PhysicalTime, NodeId, LogicalCounter) │ + └──────────────────────┬──────────────────────────┘ + │ + ┌───────────────────────────────────┼───────────────────────────────────┐ + │ │ │ + ▼ ▼ ▼ +┌───────────────┐ ┌───────────────────┐ ┌───────────────┐ +│ Scheduler │ │ Offline Merge │ │ Integration │ +│ Queue Chain │ │ Protocol │ │ Tests │ +│ │ │ │ │ │ +│ - HLC at │ │ - Local HLC │ │ - E2E tests │ +│ enqueue │ │ persistence │ │ - Benchmarks │ +│ - Chain link │ │ - Bundle export │ │ - Alerts │ +│ computation │ │ - Deterministic │ │ - Docs │ +│ - Batch │ │ merge │ │ │ +│ snapshots │ │ - Conflict │ │ │ +│ │ │ resolution │ │ │ +└───────────────┘ └───────────────────┘ └───────────────┘ +``` + +## Sprint Breakdown + +| Sprint | Module | Scope | Est. Effort | +|--------|--------|-------|-------------| +| [002_001](SPRINT_20260105_002_001_LB_hlc_core_library.md) | Library | HLC core implementation | 3 days | +| [002_002](SPRINT_20260105_002_002_SCHEDULER_hlc_queue_chain.md) | Scheduler | Queue chain integration | 4 days | +| [002_003](SPRINT_20260105_002_003_ROUTER_hlc_offline_merge.md) | Router/AirGap | Offline merge protocol | 4 days | +| [002_004](SPRINT_20260105_002_004_BE_hlc_integration_tests.md) | Testing | Integration & E2E tests | 3 days | + +**Total Estimated Effort:** ~14 days (2-3 weeks with buffer) + +## Dependency Graph + +``` +SPRINT_20260104_001_BE (TimeProvider injection) + │ + ▼ +SPRINT_20260105_002_001_LB (HLC core library) + │ + ▼ +SPRINT_20260105_002_002_SCHEDULER (Queue chain) + │ + ▼ +SPRINT_20260105_002_003_ROUTER (Offline merge) + │ + ▼ +SPRINT_20260105_002_004_BE (Integration tests) + │ + ▼ + Production Rollout +``` + +## Task Summary + +### Sprint 002_001: HLC Core Library (12 tasks) +- HLC timestamp struct with comparison +- Tick/Receive algorithm implementation +- State persistence (PostgreSQL, in-memory) +- JSON/Npgsql serialization +- Unit tests and benchmarks + +### Sprint 002_002: Scheduler Queue Chain (22 tasks) +- Database schema: `scheduler_log`, `batch_snapshot`, `chain_heads` +- Chain link computation +- HLC-based enqueue/dequeue services +- Redis/NATS adapter updates +- Batch snapshot with DSSE signing +- Chain verification +- Feature flags for gradual rollout + +### Sprint 002_003: Offline Merge Protocol (21 tasks) +- Offline HLC manager +- File-based job log store +- Merge algorithm with total ordering +- Conflict resolution +- Air-gap bundle format +- CLI command updates (`stella airgap export/import`) +- Integration with Router transport + +### Sprint 002_004: Integration Tests (22 tasks) +- HLC propagation tests +- Chain integrity tests +- Batch snapshot + Attestor integration +- Offline sync tests +- Replay determinism tests +- Performance benchmarks +- Grafana dashboard and alerts +- Documentation updates + +## Key Design Decisions + +| Decision | Rationale | +|----------|-----------| +| HLC over Lamport | Physical time component improves debuggability | +| Separate `scheduler_log` table | Avoid breaking changes to existing `jobs` table | +| Chain link at enqueue | Ensures ordering proof exists before execution | +| Feature flags | Gradual rollout; easy rollback | +| DSSE signing optional | Not all deployments need attestation | + +## Risk Register + +| Risk | Probability | Impact | Mitigation | +|------|-------------|--------|------------| +| Performance regression | Medium | Medium | Benchmarks; feature flag for rollback | +| Clock skew exceeds tolerance | Low | High | NTP hardening; pre-sync validation | +| Migration complexity | Medium | Medium | Dual-write mode; gradual rollout | +| Chain corruption | Low | Critical | Verification alerts; immutable logs | + +## Success Criteria + +1. **Determinism:** Same inputs produce same HLC order across restarts/nodes +2. **Chain Integrity:** 100% tampering detection in verification tests +3. **Offline Merge:** Jobs from multiple offline nodes merge in correct HLC order +4. **Performance:** HLC tick > 100K/sec; chain verification < 100ms/1K entries +5. **Replay:** HLC-ordered replay produces identical results + +## Rollout Plan + +### Phase 1: Shadow Mode (Week 1) +- Deploy with `EnableHlcOrdering = false`, `DualWriteMode = true` +- HLC timestamps recorded but not used for ordering +- Verify chain integrity on shadow writes + +### Phase 2: Canary (Week 2) +- Enable `EnableHlcOrdering = true` for 5% of tenants +- Monitor metrics: latency, errors, chain verifications +- Compare results between HLC and legacy ordering + +### Phase 3: General Availability (Week 3) +- Gradual rollout to all tenants +- Disable `DualWriteMode` after 1 week of stable GA +- Deprecate legacy ordering path + +### Phase 4: Offline Features (Week 4+) +- Enable air-gap bundle export/import with HLC +- Test multi-node merge scenarios +- Document operational procedures + +## Metrics to Monitor + +``` +# HLC Health +hlc_ticks_total +hlc_clock_skew_rejections_total +hlc_physical_time_offset_seconds + +# Scheduler Chain +scheduler_hlc_enqueues_total +scheduler_chain_verifications_total +scheduler_chain_verification_failures_total +scheduler_batch_snapshots_total + +# Offline Sync +airgap_bundles_exported_total +airgap_bundles_imported_total +airgap_jobs_synced_total +airgap_merge_conflicts_total +airgap_sync_duration_seconds +``` + +## Documentation Deliverables + +- [ ] `docs/ARCHITECTURE_REFERENCE.md` - HLC section +- [ ] `docs/modules/scheduler/architecture.md` - HLC ordering +- [ ] `docs/airgap/OFFLINE_KIT.md` - HLC merge protocol +- [ ] `docs/observability/observability.md` - HLC metrics +- [ ] `docs/operations/runbooks/hlc-troubleshooting.md` +- [ ] `CLAUDE.md` Section 8.19 - HLC guidelines + +## Contact & Ownership + +- **Sprint Owner:** Guild +- **Technical Lead:** TBD +- **Review:** Architecture Board + +## References + +- Product Advisory: "Audit-safe job queue ordering using monotonic timestamps" +- Gap Analysis: StellaOps implementation vs. advisory (2026-01-05) +- HLC Paper: "Logical Physical Clocks and Consistent Snapshots" (Kulkarni et al.) diff --git a/docs/implplan/SPRINT_20260105_002_001_LB_hlc_core_library.md b/docs/implplan/SPRINT_20260105_002_001_LB_hlc_core_library.md new file mode 100644 index 000000000..e6779df20 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_001_LB_hlc_core_library.md @@ -0,0 +1,343 @@ +# Sprint 20260105_002_001_LB - HLC: Hybrid Logical Clock Core Library + +## Topic & Scope + +Implement a Hybrid Logical Clock (HLC) library for deterministic, monotonic job ordering across distributed nodes. This addresses the gap identified in the "Audit-safe job queue ordering" product advisory where StellaOps currently uses wall-clock timestamps susceptible to clock skew. + +- **Working directory:** `src/__Libraries/StellaOps.HybridLogicalClock/` +- **Evidence:** NuGet package, unit tests, integration tests, benchmark results + +## Problem Statement + +Current StellaOps architecture uses: +- `TimeProvider.GetUtcNow()` for wall-clock time (deterministic but not skew-resistant) +- Per-module sequence numbers (local ordering, not global) +- Hash chains only in downstream ledgers (Findings, Orchestrator Audit) + +The advisory prescribes: +- HLC `(T, NodeId, Ctr)` tuples for global logical time +- Total ordering via `(T_hlc, PartitionKey?, JobId)` sort key +- Hash chain at enqueue time, not just downstream + +## Dependencies & Concurrency + +- **Depends on:** SPRINT_20260104_001_BE (TimeProvider injection complete) +- **Blocks:** SPRINT_20260105_002_002_SCHEDULER (HLC queue chain) +- **Parallel safe:** Library development independent of other modules + +## Documentation Prerequisites + +- docs/README.md +- docs/ARCHITECTURE_REFERENCE.md +- CLAUDE.md Section 8.2 (Deterministic Time & ID Generation) +- Product Advisory: "Audit-safe job queue ordering using monotonic timestamps" + +## Technical Design + +### HLC Algorithm (Lamport + Physical Clock Hybrid) + +``` +On local event or send: + l' = l + l = max(l, physical_clock()) + if l == l': + c = c + 1 + else: + c = 0 + return (l, node_id, c) + +On receive(m_l, m_c): + l' = l + l = max(l', m_l, physical_clock()) + if l == l' == m_l: + c = max(c, m_c) + 1 + elif l == l': + c = c + 1 + elif l == m_l: + c = m_c + 1 + else: + c = 0 + return (l, node_id, c) +``` + +### Data Model + +```csharp +/// +/// Hybrid Logical Clock timestamp providing monotonic, causally-ordered time +/// across distributed nodes even under clock skew. +/// +public readonly record struct HlcTimestamp : IComparable +{ + /// Physical time component (Unix milliseconds UTC). + public required long PhysicalTime { get; init; } + + /// Unique node identifier (e.g., "scheduler-east-1"). + public required string NodeId { get; init; } + + /// Logical counter for events at same physical time. + public required int LogicalCounter { get; init; } + + /// String representation for storage: "1704067200000-scheduler-east-1-42" + public string ToSortableString() => $"{PhysicalTime:D13}-{NodeId}-{LogicalCounter:D6}"; + + /// Parse from sortable string format. + public static HlcTimestamp Parse(string value); + + /// Compare for total ordering. + public int CompareTo(HlcTimestamp other); +} +``` + +### Interfaces + +```csharp +/// +/// Hybrid Logical Clock for monotonic timestamp generation. +/// +public interface IHybridLogicalClock +{ + /// Generate next timestamp for local event. + HlcTimestamp Tick(); + + /// Update clock on receiving remote timestamp, return merged result. + HlcTimestamp Receive(HlcTimestamp remote); + + /// Current clock state (for persistence/recovery). + HlcTimestamp Current { get; } + + /// Node identifier for this clock instance. + string NodeId { get; } +} + +/// +/// Persistent storage for HLC state (survives restarts). +/// +public interface IHlcStateStore +{ + /// Load last persisted HLC state for node. + Task LoadAsync(string nodeId, CancellationToken ct = default); + + /// Persist HLC state (called after each tick). + Task SaveAsync(HlcTimestamp timestamp, CancellationToken ct = default); +} +``` + +### PostgreSQL Schema + +```sql +-- HLC state persistence (one row per node) +CREATE TABLE scheduler.hlc_state ( + node_id TEXT PRIMARY KEY, + physical_time BIGINT NOT NULL, + logical_counter INT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Index for recovery queries +CREATE INDEX idx_hlc_state_updated ON scheduler.hlc_state(updated_at DESC); +``` + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owner | Task Definition | +|---|---------|--------|------------|-------|-----------------| +| 1 | HLC-001 | TODO | - | Guild | Create `StellaOps.HybridLogicalClock` project with Directory.Build.props integration | +| 2 | HLC-002 | TODO | HLC-001 | Guild | Implement `HlcTimestamp` record with comparison, parsing, serialization | +| 3 | HLC-003 | TODO | HLC-002 | Guild | Implement `HybridLogicalClock` class with Tick/Receive/Current | +| 4 | HLC-004 | TODO | HLC-003 | Guild | Implement `IHlcStateStore` interface and `InMemoryHlcStateStore` | +| 5 | HLC-005 | TODO | HLC-004 | Guild | Implement `PostgresHlcStateStore` with atomic update semantics | +| 6 | HLC-006 | TODO | HLC-003 | Guild | Add `HlcTimestampJsonConverter` for System.Text.Json serialization | +| 7 | HLC-007 | TODO | HLC-003 | Guild | Add `HlcTimestampTypeHandler` for Npgsql/Dapper | +| 8 | HLC-008 | TODO | HLC-005 | Guild | Write unit tests: tick monotonicity, receive merge, clock skew handling | +| 9 | HLC-009 | TODO | HLC-008 | Guild | Write integration tests: concurrent ticks, node restart recovery | +| 10 | HLC-010 | TODO | HLC-009 | Guild | Write benchmarks: tick throughput, memory allocation | +| 11 | HLC-011 | TODO | HLC-010 | Guild | Create `HlcServiceCollectionExtensions` for DI registration | +| 12 | HLC-012 | TODO | HLC-011 | Guild | Documentation: README.md, API docs, usage examples | + +## Implementation Details + +### Clock Skew Tolerance + +```csharp +public class HybridLogicalClock : IHybridLogicalClock +{ + private readonly TimeProvider _timeProvider; + private readonly string _nodeId; + private readonly IHlcStateStore _stateStore; + private readonly TimeSpan _maxClockSkew; + + private long _lastPhysicalTime; + private int _logicalCounter; + private readonly object _lock = new(); + + public HybridLogicalClock( + TimeProvider timeProvider, + string nodeId, + IHlcStateStore stateStore, + TimeSpan? maxClockSkew = null) + { + _timeProvider = timeProvider; + _nodeId = nodeId; + _stateStore = stateStore; + _maxClockSkew = maxClockSkew ?? TimeSpan.FromMinutes(1); + } + + public HlcTimestamp Tick() + { + lock (_lock) + { + var physicalNow = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); + + if (physicalNow > _lastPhysicalTime) + { + _lastPhysicalTime = physicalNow; + _logicalCounter = 0; + } + else + { + _logicalCounter++; + } + + var timestamp = new HlcTimestamp + { + PhysicalTime = _lastPhysicalTime, + NodeId = _nodeId, + LogicalCounter = _logicalCounter + }; + + // Persist state asynchronously (fire-and-forget with error logging) + _ = _stateStore.SaveAsync(timestamp); + + return timestamp; + } + } + + public HlcTimestamp Receive(HlcTimestamp remote) + { + lock (_lock) + { + var physicalNow = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds(); + + // Validate clock skew + var skew = TimeSpan.FromMilliseconds(Math.Abs(remote.PhysicalTime - physicalNow)); + if (skew > _maxClockSkew) + { + throw new HlcClockSkewException(skew, _maxClockSkew); + } + + var maxPhysical = Math.Max(Math.Max(_lastPhysicalTime, remote.PhysicalTime), physicalNow); + + if (maxPhysical == _lastPhysicalTime && maxPhysical == remote.PhysicalTime) + { + _logicalCounter = Math.Max(_logicalCounter, remote.LogicalCounter) + 1; + } + else if (maxPhysical == _lastPhysicalTime) + { + _logicalCounter++; + } + else if (maxPhysical == remote.PhysicalTime) + { + _logicalCounter = remote.LogicalCounter + 1; + } + else + { + _logicalCounter = 0; + } + + _lastPhysicalTime = maxPhysical; + + return new HlcTimestamp + { + PhysicalTime = _lastPhysicalTime, + NodeId = _nodeId, + LogicalCounter = _logicalCounter + }; + } + } +} +``` + +### Comparison for Total Ordering + +```csharp +public int CompareTo(HlcTimestamp other) +{ + // Primary: physical time + var physicalCompare = PhysicalTime.CompareTo(other.PhysicalTime); + if (physicalCompare != 0) return physicalCompare; + + // Secondary: logical counter + var counterCompare = LogicalCounter.CompareTo(other.LogicalCounter); + if (counterCompare != 0) return counterCompare; + + // Tertiary: node ID (for stable tie-breaking) + return string.Compare(NodeId, other.NodeId, StringComparison.Ordinal); +} +``` + +## Test Cases + +### Unit Tests + +| Test | Description | +|------|-------------| +| `Tick_Monotonic` | Successive ticks always increase | +| `Tick_SamePhysicalTime_IncrementCounter` | Counter increments when physical time unchanged | +| `Tick_NewPhysicalTime_ResetCounter` | Counter resets when physical time advances | +| `Receive_MergesCorrectly` | Remote timestamp merged per HLC algorithm | +| `Receive_ClockSkewExceeded_Throws` | Excessive skew detected and rejected | +| `Parse_RoundTrip` | ToSortableString/Parse symmetry | +| `CompareTo_TotalOrdering` | All orderings follow spec | + +### Integration Tests + +| Test | Description | +|------|-------------| +| `ConcurrentTicks_AllUnique` | 1000 concurrent ticks produce unique timestamps | +| `NodeRestart_ResumesFromPersisted` | After restart, clock >= persisted state | +| `MultiNode_CausalOrdering` | Messages across nodes maintain causal order | +| `PostgresStateStore_AtomicUpdate` | Concurrent saves don't lose state | + +## Metrics & Observability + +```csharp +// Counters +hlc_ticks_total{node_id} // Total ticks generated +hlc_receives_total{node_id} // Total remote timestamps received +hlc_clock_skew_rejections_total{node_id} // Skew threshold exceeded + +// Histograms +hlc_tick_duration_seconds{node_id} // Tick operation latency +hlc_logical_counter_value{node_id} // Counter distribution + +// Gauges +hlc_physical_time_offset_seconds{node_id} // Drift from wall clock +``` + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Store physical time as Unix milliseconds | Sufficient precision, compact storage | +| Use string node ID (not UUID) | Human-readable, stable across restarts | +| Fire-and-forget state persistence | Performance; recovery handles gaps | +| 1-minute default max skew | Balance between strictness and operability | + +| Risk | Mitigation | +|------|------------| +| Clock skew exceeds threshold | Alert on `hlc_clock_skew_rejections_total`; NTP hardening | +| State store unavailable | In-memory continues; warns on recovery | +| Counter overflow (INT) | At 1M ticks/sec, 35 minutes to overflow; use long if needed | + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +## Next Checkpoints + +- 2026-01-06: HLC-001 to HLC-003 complete (core implementation) +- 2026-01-07: HLC-004 to HLC-007 complete (persistence + serialization) +- 2026-01-08: HLC-008 to HLC-012 complete (tests, docs, DI) diff --git a/docs/implplan/SPRINT_20260105_002_001_REPLAY_complete_replay_infrastructure.md b/docs/implplan/SPRINT_20260105_002_001_REPLAY_complete_replay_infrastructure.md new file mode 100644 index 000000000..b36620f67 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_001_REPLAY_complete_replay_infrastructure.md @@ -0,0 +1,528 @@ +# Sprint 20260105_002_001_REPLAY - Complete Replay Infrastructure + +## Topic & Scope + +Complete the existing replay infrastructure to achieve full "Verifiable Policy Replay" as described in the product advisory. This sprint focuses on wiring existing stubs, completing DSSE verification, and adding the compact replay proof format. + +**Advisory Reference:** Product advisory on deterministic replay - "Verifiable Policy Replay (deterministic time-travel)" section. + +**Key Insight:** StellaOps has ~75% of the replay infrastructure built. This sprint closes the remaining gaps by integrating existing components (VerdictBuilder, Signer) into the CLI and API, and standardizing the replay proof output format. + +**Working directory:** `src/Cli/`, `src/Replay/`, `src/__Libraries/StellaOps.Replay.Core/` + +**Evidence:** Functional `stella verify --bundle` with full replay, `stella prove --at` command, DSSE signature verification, compact `replay-proof:` format. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| KnowledgeSnapshot model | Internal | Available | +| ReplayBundleWriter | Internal | Available | +| ReplayEngine | Internal | Available | +| VerdictBuilder | Internal | Stub exists, needs integration | +| ISigner/DSSE | Internal | Available in Attestor module | +| DsseHelper | Internal | Available | + +**Parallel Execution:** Tasks RPL-001 through RPL-005 (VerdictBuilder wiring) must complete before RPL-006 (DSSE). RPL-007 through RPL-010 (CLI) can proceed in parallel once dependencies land. + +--- + +## Documentation Prerequisites + +- `docs/modules/replay/architecture.md` (if exists) +- `docs/modules/attestor/architecture.md` +- CLAUDE.md sections on determinism (8.1-8.18) +- Existing: `src/__Libraries/StellaOps.Replay.Core/Models/KnowledgeSnapshot.cs` +- Existing: `src/Policy/__Libraries/StellaOps.Policy/Replay/ReplayEngine.cs` + +--- + +## Problem Analysis + +### Current State + +After prior implementation work: +- `KnowledgeSnapshot` model captures all inputs (SBOMs, VEX, feeds, policy, seeds) +- `ReplayBundleWriter` produces deterministic `.tar.zst` bundles +- `ReplayEngine` replays with frozen inputs and compares verdicts +- `VerdictReplayEndpoints` API exists with eligibility checking +- `stella verify --bundle` CLI exists but `ReplayVerdictAsync()` returns null (stub) +- DSSE signature verification marked "not implemented" + +**Remaining Gaps:** +1. `stella verify --bundle` doesn't actually replay verdicts +2. No DSSE signature verification on bundles +3. No compact `replay-proof:` output format +4. No `stella prove --image --at ` command + +### Target Capabilities + +``` + Replay Infrastructure Complete ++------------------------------------------------------------------+ +| | +| stella verify --bundle B.dsig | +| ├── Load manifest.json | +| ├── Validate input hashes (SBOM, feeds, VEX, policy) | +| ├── Execute VerdictBuilder.ReplayAsync(manifest) <-- NEW | +| ├── Compare replayed verdict hash to expected | +| ├── Verify DSSE signature <-- NEW | +| └── Output: replay-proof: <-- NEW | +| | +| stella prove --image sha256:abc... --at 2025-12-15T10:00Z | +| ├── Query TimelineIndexer for snapshot at timestamp <-- NEW | +| ├── Fetch bundle from CAS | +| ├── Execute replay (same as verify) | +| └── Output: replay-proof: | +| | ++------------------------------------------------------------------+ +``` + +--- + +## Architecture Design + +### ReplayProof Schema + +```csharp +// src/__Libraries/StellaOps.Replay.Core/Models/ReplayProof.cs +namespace StellaOps.Replay.Core.Models; + +/// +/// Compact proof artifact for audit trails and ticket attachments. +/// +public sealed record ReplayProof +{ + /// + /// SHA-256 of the replay bundle used. + /// + public required string BundleHash { get; init; } + + /// + /// Policy version at replay time. + /// + public required string PolicyVersion { get; init; } + + /// + /// Merkle root of verdict findings. + /// + public required string VerdictRoot { get; init; } + + /// + /// Replay execution duration in milliseconds. + /// + public required long DurationMs { get; init; } + + /// + /// Whether replayed verdict matches original. + /// + public required bool VerdictMatches { get; init; } + + /// + /// UTC timestamp of replay execution. + /// + public required DateTimeOffset ReplayedAt { get; init; } + + /// + /// Engine version that performed the replay. + /// + public required string EngineVersion { get; init; } + + /// + /// Generate compact proof string for ticket/PR attachment. + /// Format: replay-proof:<base64url(sha256(canonical_json))> + /// + public string ToCompactString() + { + var canonical = CanonicalJsonSerializer.Serialize(this); + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(canonical)); + var b64 = Convert.ToBase64String(hash).Replace("+", "-").Replace("/", "_").TrimEnd('='); + return $"replay-proof:{b64}"; + } +} +``` + +### VerdictBuilder Integration + +```csharp +// src/Cli/StellaOps.Cli/Commands/CommandHandlers.VerifyBundle.cs +// Enhancement to existing ReplayVerdictAsync method + +private static async Task ReplayVerdictAsync( + IServiceProvider services, + string bundleDir, + ReplayBundleManifest manifest, + List violations, + ILogger logger, + CancellationToken cancellationToken) +{ + var verdictBuilder = services.GetService(); + if (verdictBuilder is null) + { + logger.LogWarning("VerdictBuilder not registered - replay skipped"); + violations.Add(new BundleViolation( + "verdict.replay.service_unavailable", + "VerdictBuilder service not available in DI container")); + return null; + } + + try + { + // Load frozen inputs from bundle + var sbomPath = Path.Combine(bundleDir, manifest.Inputs.Sbom.Path); + var feedsPath = manifest.Inputs.Feeds is not null + ? Path.Combine(bundleDir, manifest.Inputs.Feeds.Path) : null; + var vexPath = manifest.Inputs.Vex is not null + ? Path.Combine(bundleDir, manifest.Inputs.Vex.Path) : null; + var policyPath = manifest.Inputs.Policy is not null + ? Path.Combine(bundleDir, manifest.Inputs.Policy.Path) : null; + + var replayRequest = new VerdictReplayRequest + { + SbomPath = sbomPath, + FeedsPath = feedsPath, + VexPath = vexPath, + PolicyPath = policyPath, + ImageDigest = manifest.Scan.ImageDigest, + PolicyDigest = manifest.Scan.PolicyDigest, + FeedSnapshotDigest = manifest.Scan.FeedSnapshotDigest + }; + + var result = await verdictBuilder.ReplayAsync(replayRequest, cancellationToken) + .ConfigureAwait(false); + + if (!result.Success) + { + violations.Add(new BundleViolation( + "verdict.replay.failed", + result.Error ?? "Verdict replay failed without error message")); + return null; + } + + logger.LogInformation("Verdict replay completed: Hash={Hash}", result.VerdictHash); + return result.VerdictHash; + } + catch (Exception ex) + { + logger.LogError(ex, "Verdict replay threw exception"); + violations.Add(new BundleViolation( + "verdict.replay.exception", + $"Replay exception: {ex.Message}")); + return null; + } +} +``` + +### DSSE Verification Integration + +```csharp +// src/Cli/StellaOps.Cli/Commands/CommandHandlers.VerifyBundle.cs +// Enhancement to existing VerifyDsseSignatureAsync method + +private static async Task VerifyDsseSignatureAsync( + IServiceProvider services, + string dssePath, + string bundleDir, + List violations, + ILogger logger, + CancellationToken cancellationToken) +{ + var dsseVerifier = services.GetService(); + if (dsseVerifier is null) + { + logger.LogWarning("DSSE verifier not registered - signature verification skipped"); + violations.Add(new BundleViolation( + "signature.verify.service_unavailable", + "DSSE verifier service not available")); + return false; + } + + try + { + var envelopeJson = await File.ReadAllTextAsync(dssePath, cancellationToken) + .ConfigureAwait(false); + + // Look for public key in attestation folder + var pubKeyPath = Path.Combine(bundleDir, "attestation", "public-key.pem"); + if (!File.Exists(pubKeyPath)) + { + pubKeyPath = Path.Combine(bundleDir, "attestation", "signing-key.pub"); + } + + if (!File.Exists(pubKeyPath)) + { + violations.Add(new BundleViolation( + "signature.key.missing", + "No public key found in attestation folder")); + return false; + } + + var publicKeyPem = await File.ReadAllTextAsync(pubKeyPath, cancellationToken) + .ConfigureAwait(false); + + var result = await dsseVerifier.VerifyAsync( + envelopeJson, + publicKeyPem, + cancellationToken).ConfigureAwait(false); + + if (!result.IsValid) + { + violations.Add(new BundleViolation( + "signature.verify.invalid", + result.Error ?? "DSSE signature verification failed")); + return false; + } + + logger.LogInformation("DSSE signature verified successfully"); + return true; + } + catch (Exception ex) + { + logger.LogError(ex, "DSSE verification threw exception"); + violations.Add(new BundleViolation( + "signature.verify.exception", + $"Verification exception: {ex.Message}")); + return false; + } +} +``` + +### stella prove Command + +```csharp +// src/Cli/StellaOps.Cli/Commands/ProveCommandGroup.cs +namespace StellaOps.Cli.Commands; + +/// +/// Command group for replay proof operations. +/// +internal static class ProveCommandGroup +{ + public static Command CreateProveCommand() + { + var imageOption = new Option( + "--image", + "Image digest (sha256:...) to generate proof for") + { IsRequired = true }; + + var atOption = new Option( + "--at", + "Point-in-time for snapshot lookup (ISO 8601)"); + + var snapshotOption = new Option( + "--snapshot", + "Explicit snapshot ID to use instead of time lookup"); + + var outputOption = new Option( + "--output", + () => "compact", + "Output format: compact, json, full"); + + var command = new Command("prove", "Generate replay proof for an image verdict") + { + imageOption, + atOption, + snapshotOption, + outputOption + }; + + command.SetHandler(async (context) => + { + var image = context.ParseResult.GetValueForOption(imageOption)!; + var at = context.ParseResult.GetValueForOption(atOption); + var snapshot = context.ParseResult.GetValueForOption(snapshotOption); + var output = context.ParseResult.GetValueForOption(outputOption)!; + var ct = context.GetCancellationToken(); + + await HandleProveAsync( + context.BindingContext.GetRequiredService(), + image, at, snapshot, output, ct).ConfigureAwait(false); + }); + + return command; + } + + private static async Task HandleProveAsync( + IServiceProvider services, + string imageDigest, + DateTimeOffset? at, + string? snapshotId, + string outputFormat, + CancellationToken ct) + { + var timelineService = services.GetRequiredService(); + var bundleStore = services.GetRequiredService(); + var replayExecutor = services.GetRequiredService(); + + // Step 1: Resolve snapshot + string resolvedSnapshotId; + if (!string.IsNullOrEmpty(snapshotId)) + { + resolvedSnapshotId = snapshotId; + } + else if (at.HasValue) + { + var query = new TimelineQuery + { + ArtifactDigest = imageDigest, + PointInTime = at.Value, + EventType = TimelineEventType.VerdictComputed + }; + var result = await timelineService.QueryAsync(query, ct).ConfigureAwait(false); + if (result.Events.Count == 0) + { + AnsiConsole.MarkupLine("[red]No verdict found for image at specified time[/]"); + Environment.ExitCode = 1; + return; + } + resolvedSnapshotId = result.Events[0].SnapshotId; + } + else + { + // Use latest snapshot + var latest = await timelineService.GetLatestSnapshotAsync(imageDigest, ct) + .ConfigureAwait(false); + if (latest is null) + { + AnsiConsole.MarkupLine("[red]No snapshots found for image[/]"); + Environment.ExitCode = 1; + return; + } + resolvedSnapshotId = latest.SnapshotId; + } + + // Step 2: Fetch bundle + var bundle = await bundleStore.GetBundleAsync(resolvedSnapshotId, ct) + .ConfigureAwait(false); + if (bundle is null) + { + AnsiConsole.MarkupLine($"[red]Bundle not found for snapshot {resolvedSnapshotId}[/]"); + Environment.ExitCode = 1; + return; + } + + // Step 3: Execute replay + var replayResult = await replayExecutor.ExecuteAsync(bundle, ct).ConfigureAwait(false); + + // Step 4: Generate proof + var proof = new ReplayProof + { + BundleHash = bundle.Sha256, + PolicyVersion = bundle.Manifest.PolicyVersion, + VerdictRoot = replayResult.VerdictRoot, + DurationMs = replayResult.DurationMs, + VerdictMatches = replayResult.VerdictMatches, + ReplayedAt = DateTimeOffset.UtcNow, + EngineVersion = replayResult.EngineVersion + }; + + // Step 5: Output + switch (outputFormat.ToLowerInvariant()) + { + case "compact": + AnsiConsole.WriteLine(proof.ToCompactString()); + break; + case "json": + var json = JsonSerializer.Serialize(proof, new JsonSerializerOptions { WriteIndented = true }); + AnsiConsole.WriteLine(json); + break; + case "full": + OutputFullProof(proof, replayResult); + break; + } + } + + private static void OutputFullProof(ReplayProof proof, ReplayExecutionResult result) + { + var table = new Table().AddColumns("Field", "Value"); + table.AddRow("Bundle Hash", proof.BundleHash); + table.AddRow("Policy Version", proof.PolicyVersion); + table.AddRow("Verdict Root", proof.VerdictRoot); + table.AddRow("Duration", $"{proof.DurationMs}ms"); + table.AddRow("Verdict Matches", proof.VerdictMatches ? "[green]Yes[/]" : "[red]No[/]"); + table.AddRow("Engine Version", proof.EngineVersion); + table.AddRow("Replayed At", proof.ReplayedAt.ToString("O")); + AnsiConsole.Write(table); + + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Compact Proof:[/]"); + AnsiConsole.WriteLine(proof.ToCompactString()); + } +} +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| **VerdictBuilder Integration** | +| 1 | RPL-001 | TODO | - | Replay Guild | Define `IVerdictBuilder.ReplayAsync()` contract in `StellaOps.Verdict` | +| 2 | RPL-002 | TODO | RPL-001 | Replay Guild | Implement `VerdictBuilder.ReplayAsync()` using frozen inputs | +| 3 | RPL-003 | TODO | RPL-002 | Replay Guild | Wire `VerdictBuilder` into CLI DI container | +| 4 | RPL-004 | TODO | RPL-003 | Replay Guild | Update `CommandHandlers.VerifyBundle.ReplayVerdictAsync()` to use service | +| 5 | RPL-005 | TODO | RPL-004 | Replay Guild | Unit tests: VerdictBuilder replay with fixtures | +| **DSSE Verification** | +| 6 | RPL-006 | TODO | - | Attestor Guild | Define `IDsseVerifier` interface in `StellaOps.Attestation` | +| 7 | RPL-007 | TODO | RPL-006 | Attestor Guild | Implement `DsseVerifier` using existing `DsseHelper` | +| 8 | RPL-008 | TODO | RPL-007 | CLI Guild | Wire `DsseVerifier` into CLI DI container | +| 9 | RPL-009 | TODO | RPL-008 | CLI Guild | Update `CommandHandlers.VerifyBundle.VerifyDsseSignatureAsync()` | +| 10 | RPL-010 | TODO | RPL-009 | Attestor Guild | Unit tests: DSSE verification with valid/invalid signatures | +| **ReplayProof Schema** | +| 11 | RPL-011 | TODO | - | Replay Guild | Create `ReplayProof` model in `StellaOps.Replay.Core` | +| 12 | RPL-012 | TODO | RPL-011 | Replay Guild | Implement `ToCompactString()` with canonical JSON + SHA-256 | +| 13 | RPL-013 | TODO | RPL-012 | Replay Guild | Update `stella verify --bundle` to output replay proof | +| 14 | RPL-014 | TODO | RPL-013 | Replay Guild | Unit tests: Replay proof generation and parsing | +| **stella prove Command** | +| 15 | RPL-015 | TODO | RPL-011 | CLI Guild | Create `ProveCommandGroup.cs` with command structure | +| 16 | RPL-016 | TODO | RPL-015 | CLI Guild | Implement `ITimelineQueryService` adapter for snapshot lookup | +| 17 | RPL-017 | TODO | RPL-016 | CLI Guild | Implement `IReplayBundleStore` adapter for bundle retrieval | +| 18 | RPL-018 | TODO | RPL-017 | CLI Guild | Wire `stella prove` into main command tree | +| 19 | RPL-019 | TODO | RPL-018 | CLI Guild | Integration tests: `stella prove` with test bundles | +| **Documentation & Polish** | +| 20 | RPL-020 | TODO | RPL-019 | Docs Guild | Update `docs/cli/admin-reference.md` with new commands | +| 21 | RPL-021 | TODO | RPL-020 | Docs Guild | Create `docs/modules/replay/replay-proof-schema.md` | +| 22 | RPL-022 | TODO | RPL-021 | QA Guild | E2E test: Full verify → prove workflow | + +--- + +## Success Metrics + +| Metric | Before | After | Target | +|--------|--------|-------|--------| +| `stella verify --bundle` actually replays | No | Yes | 100% | +| DSSE signature verification functional | No | Yes | 100% | +| Compact replay proof format available | No | Yes | 100% | +| `stella prove --at` command available | No | Yes | 100% | +| Replay latency (warm cache) | N/A | <5s | <5s (p50) | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| VerdictBuilder may not be ready | Risk | Fall back to existing ReplayEngine, document limitations | +| DSSE verification requires key management | Constraint | Use embedded public key in bundle, document key rotation | +| Timeline service may not support point-in-time queries | Risk | Add snapshot-by-timestamp index to Postgres | +| Compact proof format needs to be tamper-evident | Decision | Include SHA-256 of canonical JSON, not just fields | + +--- + +## Next Checkpoints + +- RPL-001 through RPL-005 (VerdictBuilder) target completion +- RPL-006 through RPL-010 (DSSE) target completion +- RPL-015 through RPL-019 (stella prove) target completion +- RPL-022 (E2E) sprint completion gate diff --git a/docs/implplan/SPRINT_20260105_002_002_FACET_abstraction_layer.md b/docs/implplan/SPRINT_20260105_002_002_FACET_abstraction_layer.md new file mode 100644 index 000000000..8098ae872 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_002_FACET_abstraction_layer.md @@ -0,0 +1,676 @@ +# Sprint 20260105_002_002_FACET - Facet Abstraction Layer + +## Topic & Scope + +Implement the foundational "Facet" abstraction layer that enables per-facet sealing and drift tracking. This sprint defines the core domain models (`IFacet`, `FacetSeal`, `FacetDrift`), facet taxonomy, and per-facet Merkle tree computation. + +**Advisory Reference:** Product advisory on facet sealing - "Facet Sealing & Drift Quotas" section. + +**Key Insight:** A "facet" is a declared slice of an image (OS packages, language dependencies, key binaries, config files). By sealing facets individually with Merkle roots, we can track drift at granular levels and apply different quotas to different component types. + +**Working directory:** `src/__Libraries/StellaOps.Facet/`, `src/Scanner/` + +**Evidence:** New `StellaOps.Facet` library with models, Merkle tree computation, and facet extractors integrated into Scanner surface manifest. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| SurfaceManifestDocument | Internal | Available | +| Scanner Analyzers (OS, Lang, Native) | Internal | Available | +| Merkle tree utilities | Internal | Partial (single root exists) | +| ICryptoHash | Internal | Available | + +**Parallel Execution:** FCT-001 through FCT-010 (core models) can proceed independently. FCT-011 through FCT-020 (extractors) depend on models. FCT-021 through FCT-025 (integration) depend on extractors. + +--- + +## Documentation Prerequisites + +- `docs/modules/scanner/architecture.md` +- `src/Scanner/__Libraries/StellaOps.Scanner.Surface.FS/SurfaceManifestModels.cs` +- CLAUDE.md determinism rules +- Product advisory on facet sealing + +--- + +## Problem Analysis + +### Current State + +StellaOps currently: +- Computes a single `DeterminismMerkleRoot` for the entire scan output +- Tracks drift at aggregate level via `FnDriftCalculator` +- Has language-specific analyzers (DotNet, Node, Python, etc.) +- Has native analyzer for ELF/PE/Mach-O binaries +- No concept of "facets" as distinct trackable units + +**Gaps:** +1. No facet taxonomy or abstraction +2. No per-facet Merkle roots +3. No facet-specific file selectors +4. Cannot apply different drift quotas to different component types + +### Target Capabilities + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Facet Abstraction Layer │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ Facet Taxonomy │ │ +│ │ │ │ +│ │ OS Packages Language Dependencies Binaries │ │ +│ │ ├─ dpkg/deb ├─ npm/node_modules ├─ /usr/bin/* │ │ +│ │ ├─ rpm ├─ pip/site-packages ├─ /usr/lib/*.so │ │ +│ │ ├─ apk ├─ nuget/packages ├─ *.dll │ │ +│ │ └─ pacman ├─ maven/m2 └─ *.dylib │ │ +│ │ ├─ cargo/registry │ │ +│ │ Config Files └─ go/pkg Certificates │ │ +│ │ ├─ /etc/* ├─ /etc/ssl/certs │ │ +│ │ ├─ *.conf Interpreters ├─ /etc/pki │ │ +│ │ └─ *.yaml ├─ /usr/bin/python* └─ trust anchors │ │ +│ │ ├─ /usr/bin/node │ │ +│ │ └─ /usr/bin/ruby │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ FacetSeal Structure │ │ +│ │ │ │ +│ │ { │ │ +│ │ "imageDigest": "sha256:abc...", │ │ +│ │ "createdAt": "2026-01-05T12:00:00Z", │ │ +│ │ "facets": [ │ │ +│ │ { │ │ +│ │ "name": "os-packages", │ │ +│ │ "selector": "/var/lib/dpkg/status", │ │ +│ │ "merkleRoot": "sha256:...", │ │ +│ │ "fileCount": 1247, │ │ +│ │ "totalBytes": 15_000_000 │ │ +│ │ }, │ │ +│ │ { │ │ +│ │ "name": "lang-deps-npm", │ │ +│ │ "selector": "**/node_modules/**/package.json", │ │ +│ │ "merkleRoot": "sha256:...", │ │ +│ │ "fileCount": 523, │ │ +│ │ "totalBytes": 45_000_000 │ │ +│ │ } │ │ +│ │ ], │ │ +│ │ "signature": "DSSE envelope" │ │ +│ │ } │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Architecture Design + +### Core Facet Models + +```csharp +// src/__Libraries/StellaOps.Facet/IFacet.cs +namespace StellaOps.Facet; + +/// +/// Represents a trackable slice of an image. +/// +public interface IFacet +{ + /// + /// Unique identifier for this facet type. + /// + string FacetId { get; } + + /// + /// Human-readable name. + /// + string Name { get; } + + /// + /// Facet category (os-packages, lang-deps, binaries, config, certs). + /// + FacetCategory Category { get; } + + /// + /// Glob patterns or path selectors for files in this facet. + /// + IReadOnlyList Selectors { get; } + + /// + /// Priority for conflict resolution when files match multiple facets. + /// Lower = higher priority. + /// + int Priority { get; } +} + +public enum FacetCategory +{ + OsPackages, + LanguageDependencies, + Binaries, + Configuration, + Certificates, + Interpreters, + Custom +} +``` + +### Facet Seal Model + +```csharp +// src/__Libraries/StellaOps.Facet/FacetSeal.cs +namespace StellaOps.Facet; + +/// +/// Sealed manifest of facets for an image at a point in time. +/// +public sealed record FacetSeal +{ + /// + /// Schema version for forward compatibility. + /// + public string SchemaVersion { get; init; } = "1.0.0"; + + /// + /// Image this seal applies to. + /// + public required string ImageDigest { get; init; } + + /// + /// When the seal was created. + /// + public required DateTimeOffset CreatedAt { get; init; } + + /// + /// Build attestation reference (in-toto provenance). + /// + public string? BuildAttestationRef { get; init; } + + /// + /// Individual facet seals. + /// + public required ImmutableArray Facets { get; init; } + + /// + /// Quota configuration per facet. + /// + public ImmutableDictionary? Quotas { get; init; } + + /// + /// Combined Merkle root of all facet roots (for single-value integrity check). + /// + public required string CombinedMerkleRoot { get; init; } + + /// + /// DSSE signature over canonical form. + /// + public string? Signature { get; init; } +} + +public sealed record FacetEntry +{ + /// + /// Facet identifier (e.g., "os-packages-dpkg", "lang-deps-npm"). + /// + public required string FacetId { get; init; } + + /// + /// Human-readable name. + /// + public required string Name { get; init; } + + /// + /// Category for grouping. + /// + public required FacetCategory Category { get; init; } + + /// + /// Selectors used to identify files in this facet. + /// + public required ImmutableArray Selectors { get; init; } + + /// + /// Merkle root of all files in this facet. + /// + public required string MerkleRoot { get; init; } + + /// + /// Number of files in this facet. + /// + public required int FileCount { get; init; } + + /// + /// Total bytes across all files. + /// + public required long TotalBytes { get; init; } + + /// + /// Optional: individual file entries (for detailed audit). + /// + public ImmutableArray? Files { get; init; } +} + +public sealed record FacetFileEntry( + string Path, + string Digest, + long SizeBytes, + DateTimeOffset? ModifiedAt); + +public sealed record FacetQuota +{ + /// + /// Maximum allowed churn percentage (0-100). + /// + public decimal MaxChurnPercent { get; init; } = 10m; + + /// + /// Maximum number of changed files before alert. + /// + public int MaxChangedFiles { get; init; } = 50; + + /// + /// Glob patterns for files exempt from quota enforcement. + /// + public ImmutableArray AllowlistGlobs { get; init; } = []; + + /// + /// Action when quota exceeded: Warn, Block, RequireVex. + /// + public QuotaExceededAction Action { get; init; } = QuotaExceededAction.Warn; +} + +public enum QuotaExceededAction +{ + Warn, + Block, + RequireVex +} +``` + +### Facet Drift Model + +```csharp +// src/__Libraries/StellaOps.Facet/FacetDrift.cs +namespace StellaOps.Facet; + +/// +/// Drift detection result for a single facet. +/// +public sealed record FacetDrift +{ + /// + /// Facet this drift applies to. + /// + public required string FacetId { get; init; } + + /// + /// Files added since baseline. + /// + public required ImmutableArray Added { get; init; } + + /// + /// Files removed since baseline. + /// + public required ImmutableArray Removed { get; init; } + + /// + /// Files modified since baseline. + /// + public required ImmutableArray Modified { get; init; } + + /// + /// Drift score (0-100, higher = more drift). + /// + public required decimal DriftScore { get; init; } + + /// + /// Quota evaluation result. + /// + public required QuotaVerdict QuotaVerdict { get; init; } + + /// + /// Churn percentage = (added + removed + modified) / baseline count * 100. + /// + public decimal ChurnPercent => BaselineFileCount > 0 + ? (Added.Length + Removed.Length + Modified.Length) / (decimal)BaselineFileCount * 100 + : 0; + + /// + /// Number of files in baseline facet seal. + /// + public required int BaselineFileCount { get; init; } +} + +public sealed record FacetFileModification( + string Path, + string PreviousDigest, + string CurrentDigest, + long PreviousSizeBytes, + long CurrentSizeBytes); + +public enum QuotaVerdict +{ + Ok, + Warning, + Blocked, + RequiresVex +} +``` + +### Facet Merkle Tree + +```csharp +// src/__Libraries/StellaOps.Facet/FacetMerkleTree.cs +namespace StellaOps.Facet; + +/// +/// Computes Merkle roots for facet file sets. +/// +public sealed class FacetMerkleTree +{ + private readonly ICryptoHash _cryptoHash; + + public FacetMerkleTree(ICryptoHash cryptoHash) + { + _cryptoHash = cryptoHash ?? throw new ArgumentNullException(nameof(cryptoHash)); + } + + /// + /// Compute Merkle root from file entries. + /// + public string ComputeRoot(IEnumerable files) + { + // Sort files by path for determinism + var sortedFiles = files + .OrderBy(f => f.Path, StringComparer.Ordinal) + .ToList(); + + if (sortedFiles.Count == 0) + { + // Empty tree root = SHA-256 of empty string + return "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + } + + // Build leaf nodes: hash of (path + digest + size) + var leaves = sortedFiles + .Select(f => ComputeLeafHash(f)) + .ToList(); + + // Build tree bottom-up + return ComputeMerkleRoot(leaves); + } + + private byte[] ComputeLeafHash(FacetFileEntry file) + { + // Canonical leaf format: "path|digest|size" + var canonical = $"{file.Path}|{file.Digest}|{file.SizeBytes}"; + return _cryptoHash.ComputeHash( + System.Text.Encoding.UTF8.GetBytes(canonical), + "SHA256"); + } + + private string ComputeMerkleRoot(List nodes) + { + while (nodes.Count > 1) + { + var nextLevel = new List(); + + for (var i = 0; i < nodes.Count; i += 2) + { + if (i + 1 < nodes.Count) + { + // Combine two nodes + var combined = new byte[nodes[i].Length + nodes[i + 1].Length]; + nodes[i].CopyTo(combined, 0); + nodes[i + 1].CopyTo(combined, nodes[i].Length); + nextLevel.Add(_cryptoHash.ComputeHash(combined, "SHA256")); + } + else + { + // Odd node: promote as-is + nextLevel.Add(nodes[i]); + } + } + + nodes = nextLevel; + } + + return $"sha256:{Convert.ToHexString(nodes[0]).ToLowerInvariant()}"; + } + + /// + /// Compute combined root from multiple facet roots. + /// + public string ComputeCombinedRoot(IEnumerable facets) + { + var facetRoots = facets + .OrderBy(f => f.FacetId, StringComparer.Ordinal) + .Select(f => HexToBytes(f.MerkleRoot.Replace("sha256:", ""))) + .ToList(); + + return ComputeMerkleRoot(facetRoots); + } + + private static byte[] HexToBytes(string hex) + { + return Convert.FromHexString(hex); + } +} +``` + +### Built-in Facet Definitions + +```csharp +// src/__Libraries/StellaOps.Facet/BuiltInFacets.cs +namespace StellaOps.Facet; + +/// +/// Built-in facet definitions for common image components. +/// +public static class BuiltInFacets +{ + public static IReadOnlyList All { get; } = new IFacet[] + { + // OS Package Managers + new FacetDefinition("os-packages-dpkg", "Debian Packages", FacetCategory.OsPackages, + ["/var/lib/dpkg/status", "/var/lib/dpkg/info/**"], priority: 10), + new FacetDefinition("os-packages-rpm", "RPM Packages", FacetCategory.OsPackages, + ["/var/lib/rpm/**", "/usr/lib/sysimage/rpm/**"], priority: 10), + new FacetDefinition("os-packages-apk", "Alpine Packages", FacetCategory.OsPackages, + ["/lib/apk/db/**"], priority: 10), + + // Language Dependencies + new FacetDefinition("lang-deps-npm", "NPM Packages", FacetCategory.LanguageDependencies, + ["**/node_modules/**/package.json", "**/package-lock.json"], priority: 20), + new FacetDefinition("lang-deps-pip", "Python Packages", FacetCategory.LanguageDependencies, + ["**/site-packages/**/*.dist-info/METADATA", "**/requirements.txt"], priority: 20), + new FacetDefinition("lang-deps-nuget", "NuGet Packages", FacetCategory.LanguageDependencies, + ["**/*.deps.json", "**/*.nuget/**"], priority: 20), + new FacetDefinition("lang-deps-maven", "Maven Packages", FacetCategory.LanguageDependencies, + ["**/.m2/repository/**/*.pom"], priority: 20), + new FacetDefinition("lang-deps-cargo", "Cargo Packages", FacetCategory.LanguageDependencies, + ["**/.cargo/registry/**", "**/Cargo.lock"], priority: 20), + new FacetDefinition("lang-deps-go", "Go Modules", FacetCategory.LanguageDependencies, + ["**/go.sum", "**/go/pkg/mod/**"], priority: 20), + + // Binaries + new FacetDefinition("binaries-usr", "System Binaries", FacetCategory.Binaries, + ["/usr/bin/*", "/usr/sbin/*", "/bin/*", "/sbin/*"], priority: 30), + new FacetDefinition("binaries-lib", "Shared Libraries", FacetCategory.Binaries, + ["/usr/lib/**/*.so*", "/lib/**/*.so*", "/usr/lib64/**/*.so*"], priority: 30), + + // Interpreters + new FacetDefinition("interpreters", "Language Interpreters", FacetCategory.Interpreters, + ["/usr/bin/python*", "/usr/bin/node*", "/usr/bin/ruby*", "/usr/bin/perl*"], priority: 15), + + // Configuration + new FacetDefinition("config-etc", "System Configuration", FacetCategory.Configuration, + ["/etc/**/*.conf", "/etc/**/*.cfg", "/etc/**/*.yaml", "/etc/**/*.json"], priority: 40), + + // Certificates + new FacetDefinition("certs-system", "System Certificates", FacetCategory.Certificates, + ["/etc/ssl/certs/**", "/etc/pki/**", "/usr/share/ca-certificates/**"], priority: 25), + }; + + public static IFacet? GetById(string facetId) + => All.FirstOrDefault(f => f.FacetId == facetId); +} + +internal sealed class FacetDefinition : IFacet +{ + public string FacetId { get; } + public string Name { get; } + public FacetCategory Category { get; } + public IReadOnlyList Selectors { get; } + public int Priority { get; } + + public FacetDefinition( + string facetId, + string name, + FacetCategory category, + string[] selectors, + int priority) + { + FacetId = facetId; + Name = name; + Category = category; + Selectors = selectors; + Priority = priority; + } +} +``` + +### Facet Extractor Interface + +```csharp +// src/__Libraries/StellaOps.Facet/IFacetExtractor.cs +namespace StellaOps.Facet; + +/// +/// Extracts facet file entries from an image filesystem. +/// +public interface IFacetExtractor +{ + /// + /// Extract files matching a facet's selectors. + /// + Task ExtractAsync( + IFacet facet, + IImageFileSystem imageFs, + FacetExtractionOptions? options = null, + CancellationToken ct = default); +} + +public sealed record FacetExtractionResult( + string FacetId, + ImmutableArray Files, + long TotalBytes, + TimeSpan Duration, + ImmutableArray? Errors); + +public sealed record FacetExtractionOptions +{ + /// + /// Include file content hashes (slower but required for sealing). + /// + public bool ComputeHashes { get; init; } = true; + + /// + /// Maximum files to extract per facet (0 = unlimited). + /// + public int MaxFiles { get; init; } = 0; + + /// + /// Skip files larger than this size. + /// + public long MaxFileSizeBytes { get; init; } = 100 * 1024 * 1024; // 100MB + + /// + /// Follow symlinks when extracting. + /// + public bool FollowSymlinks { get; init; } = false; +} +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| **Core Models** | +| 1 | FCT-001 | TODO | - | Facet Guild | Create `StellaOps.Facet` project structure | +| 2 | FCT-002 | TODO | FCT-001 | Facet Guild | Define `IFacet` interface and `FacetCategory` enum | +| 3 | FCT-003 | TODO | FCT-002 | Facet Guild | Define `FacetSeal` model with entries and quotas | +| 4 | FCT-004 | TODO | FCT-003 | Facet Guild | Define `FacetDrift` model with change tracking | +| 5 | FCT-005 | TODO | FCT-004 | Facet Guild | Define `FacetQuota` model with actions | +| 6 | FCT-006 | TODO | FCT-005 | Facet Guild | Unit tests: Model serialization round-trips | +| **Merkle Tree** | +| 7 | FCT-007 | TODO | FCT-003 | Facet Guild | Implement `FacetMerkleTree` with leaf computation | +| 8 | FCT-008 | TODO | FCT-007 | Facet Guild | Implement combined root from multiple facets | +| 9 | FCT-009 | TODO | FCT-008 | Facet Guild | Unit tests: Merkle root determinism | +| 10 | FCT-010 | TODO | FCT-009 | Facet Guild | Golden tests: Known inputs → known roots | +| **Built-in Facets** | +| 11 | FCT-011 | TODO | FCT-002 | Facet Guild | Define OS package facets (dpkg, rpm, apk) | +| 12 | FCT-012 | TODO | FCT-011 | Facet Guild | Define language dependency facets (npm, pip, etc.) | +| 13 | FCT-013 | TODO | FCT-012 | Facet Guild | Define binary facets (usr/bin, libs) | +| 14 | FCT-014 | TODO | FCT-013 | Facet Guild | Define config and certificate facets | +| 15 | FCT-015 | TODO | FCT-014 | Facet Guild | Create `BuiltInFacets` registry | +| **Extraction** | +| 16 | FCT-016 | TODO | FCT-015 | Scanner Guild | Define `IFacetExtractor` interface | +| 17 | FCT-017 | TODO | FCT-016 | Scanner Guild | Implement `GlobFacetExtractor` for selector matching | +| 18 | FCT-018 | TODO | FCT-017 | Scanner Guild | Integrate with Scanner's `IImageFileSystem` | +| 19 | FCT-019 | TODO | FCT-018 | Scanner Guild | Unit tests: Extraction from mock FS | +| 20 | FCT-020 | TODO | FCT-019 | Scanner Guild | Integration tests: Extraction from real image layers | +| **Surface Manifest Integration** | +| 21 | FCT-021 | TODO | FCT-020 | Scanner Guild | Add `FacetSeals` property to `SurfaceManifestDocument` | +| 22 | FCT-022 | TODO | FCT-021 | Scanner Guild | Compute facet seals during scan surface publishing | +| 23 | FCT-023 | TODO | FCT-022 | Scanner Guild | Store facet seals in Postgres alongside surface manifest | +| 24 | FCT-024 | TODO | FCT-023 | Scanner Guild | Unit tests: Surface manifest with facets | +| 25 | FCT-025 | TODO | FCT-024 | QA Guild | E2E test: Scan → facet seal generation | + +--- + +## Success Metrics + +| Metric | Before | After | Target | +|--------|--------|-------|--------| +| Per-facet Merkle roots available | No | Yes | 100% | +| Facet taxonomy defined | No | Yes | 15+ facet types | +| Facet extraction from images | No | Yes | All built-in facets | +| Surface manifest includes facets | No | Yes | 100% | +| Merkle computation deterministic | N/A | Yes | 100% reproducible | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| Facet selectors may overlap | Decision | Use priority field, document conflict resolution | +| Large images may have many files | Risk | Add MaxFiles limit, streaming extraction | +| Merkle computation adds scan latency | Trade-off | Make facet sealing opt-in via config | +| Glob matching performance | Risk | Use optimized glob library (DotNet.Glob) | +| Symlink handling complexity | Decision | Default to not following, document rationale | + +--- + +## Next Checkpoints + +- FCT-001 through FCT-006 (core models) target completion +- FCT-007 through FCT-010 (Merkle) target completion +- FCT-016 through FCT-020 (extraction) target completion +- FCT-025 (E2E) sprint completion gate diff --git a/docs/implplan/SPRINT_20260105_002_002_SCHEDULER_hlc_queue_chain.md b/docs/implplan/SPRINT_20260105_002_002_SCHEDULER_hlc_queue_chain.md new file mode 100644 index 000000000..d1afeb80f --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_002_SCHEDULER_hlc_queue_chain.md @@ -0,0 +1,427 @@ +# Sprint 20260105_002_002_SCHEDULER - HLC: Scheduler Queue Chain Integration + +## Topic & Scope + +Integrate Hybrid Logical Clock (HLC) into the Scheduler queue with cryptographic sequence proofs at enqueue time. This implements the core advisory requirement: "derive order from deterministic, monotonic time inside your system and prove the sequence with hashes." + +- **Working directory:** `src/Scheduler/` +- **Evidence:** Updated schema, queue implementations, chain verification tests + +## Problem Statement + +Current Scheduler queue implementation: +- Orders by `(priority DESC, created_at ASC, id)` - wall-clock based +- No hash chain at enqueue - chains only exist in downstream ledgers +- Redis/NATS sequences provide local ordering, not global HLC ordering + +Advisory requires: +- HLC timestamp assigned at enqueue: `t_hlc = hlc.Tick()` +- Chain link computed: `link = Hash(prev_link || job_id || t_hlc || payload_hash)` +- Total order persisted at enqueue, not dequeue + +## Dependencies & Concurrency + +- **Depends on:** SPRINT_20260105_002_001_LB (HLC core library) +- **Blocks:** SPRINT_20260105_002_003_ROUTER (offline merge protocol) +- **Parallel safe:** Scheduler-only changes; no cross-module conflicts + +## Documentation Prerequisites + +- docs/modules/scheduler/architecture.md +- src/Scheduler/AGENTS.md +- SPRINT_20260105_002_001_LB (HLC library design) +- Product Advisory: scheduler_log table specification + +## Technical Design + +### Database Schema Changes + +```sql +-- New: Scheduler log table for HLC-ordered, chain-linked jobs +CREATE TABLE scheduler.scheduler_log ( + seq_bigint BIGSERIAL PRIMARY KEY, -- Storage order (not authoritative) + tenant_id TEXT NOT NULL, + t_hlc TEXT NOT NULL, -- HLC timestamp string + partition_key TEXT, -- Optional queue partition + job_id UUID NOT NULL, + payload_hash BYTEA NOT NULL, -- SHA-256 of canonical payload + prev_link BYTEA, -- Previous chain link (null for first) + link BYTEA NOT NULL, -- Hash(prev_link || job_id || t_hlc || payload_hash) + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_scheduler_log_order UNIQUE (tenant_id, t_hlc, partition_key, job_id) +); + +CREATE INDEX idx_scheduler_log_tenant_hlc ON scheduler.scheduler_log(tenant_id, t_hlc); +CREATE INDEX idx_scheduler_log_partition ON scheduler.scheduler_log(tenant_id, partition_key, t_hlc); + +-- New: Batch snapshot table for audit anchors +CREATE TABLE scheduler.batch_snapshot ( + batch_id UUID PRIMARY KEY, + tenant_id TEXT NOT NULL, + range_start_t TEXT NOT NULL, -- HLC range start + range_end_t TEXT NOT NULL, -- HLC range end + head_link BYTEA NOT NULL, -- Chain head at snapshot + job_count INT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + signed_by TEXT, -- Optional key ID for DSSE + signature BYTEA -- Optional DSSE signature +); + +CREATE INDEX idx_batch_snapshot_tenant ON scheduler.batch_snapshot(tenant_id, created_at DESC); + +-- New: Per-partition chain head tracking +CREATE TABLE scheduler.chain_heads ( + tenant_id TEXT NOT NULL, + partition_key TEXT NOT NULL DEFAULT '', + last_link BYTEA NOT NULL, + last_t_hlc TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (tenant_id, partition_key) +); +``` + +### Chain Link Computation + +```csharp +public static class SchedulerChainLinking +{ + /// + /// Compute chain link per advisory specification: + /// link_i = Hash(link_{i-1} || job_id || t_hlc || payload_hash) + /// + public static byte[] ComputeLink( + byte[]? prevLink, + Guid jobId, + HlcTimestamp tHlc, + byte[] payloadHash) + { + using var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + // Previous link (or 32 zero bytes for first entry) + hasher.AppendData(prevLink ?? new byte[32]); + + // Job ID as bytes (big-endian for consistency) + hasher.AppendData(jobId.ToByteArray()); + + // HLC timestamp as UTF-8 bytes + hasher.AppendData(Encoding.UTF8.GetBytes(tHlc.ToSortableString())); + + // Payload hash + hasher.AppendData(payloadHash); + + return hasher.GetHashAndReset(); + } + + /// + /// Compute deterministic payload hash from canonical JSON. + /// + public static byte[] ComputePayloadHash(object payload) + { + var canonical = CanonicalJsonSerializer.Serialize(payload); + return SHA256.HashData(Encoding.UTF8.GetBytes(canonical)); + } +} +``` + +### Enqueue Flow Enhancement + +```csharp +public sealed class HlcSchedulerEnqueueService +{ + private readonly IHybridLogicalClock _hlc; + private readonly ISchedulerLogRepository _logRepository; + private readonly IChainHeadRepository _chainHeadRepository; + private readonly IGuidProvider _guidProvider; + + public async Task EnqueueAsync( + SchedulerJobPayload payload, + CancellationToken ct = default) + { + // 1. Generate HLC timestamp + var tHlc = _hlc.Tick(); + + // 2. Compute deterministic job ID from payload (idempotency) + var jobId = ComputeDeterministicJobId(payload); + + // 3. Compute payload hash + var payloadHash = SchedulerChainLinking.ComputePayloadHash(payload); + + // 4. Get previous chain link + var prevLink = await _chainHeadRepository.GetLastLinkAsync( + payload.TenantId, + payload.PartitionKey, + ct); + + // 5. Compute new chain link + var link = SchedulerChainLinking.ComputeLink(prevLink, jobId, tHlc, payloadHash); + + // 6. Insert log entry (atomic with chain head update) + await _logRepository.InsertWithChainUpdateAsync( + new SchedulerLogEntry + { + TenantId = payload.TenantId, + THlc = tHlc.ToSortableString(), + PartitionKey = payload.PartitionKey, + JobId = jobId, + PayloadHash = payloadHash, + PrevLink = prevLink, + Link = link + }, + ct); + + return new SchedulerEnqueueResult(tHlc, jobId, link); + } + + private Guid ComputeDeterministicJobId(SchedulerJobPayload payload) + { + // GUID v5 (SHA-1 based) over canonical JSON for determinism + var canonical = CanonicalJsonSerializer.Serialize(payload); + return GuidUtility.Create( + SchedulerNamespaces.JobPayload, + canonical, + version: 5); + } +} +``` + +### Dequeue with HLC Ordering + +```csharp +public sealed class HlcSchedulerDequeueService +{ + private readonly ISchedulerLogRepository _logRepository; + private readonly IJobVerdictRepository _verdictRepository; + + public async Task> DequeueAsync( + string tenantId, + string? partitionKey, + int limit, + CancellationToken ct = default) + { + // Query by HLC order (ascending) for deterministic dequeue + var logEntries = await _logRepository.GetByHlcOrderAsync( + tenantId, + partitionKey, + limit, + ct); + + var jobs = new List(); + foreach (var entry in logEntries) + { + // Check idempotency: skip if verdict already exists + var verdict = await _verdictRepository.GetAsync(entry.JobId, ct); + if (verdict is not null) + { + continue; // Already processed + } + + jobs.Add(MapToJob(entry)); + } + + return jobs; + } +} +``` + +### Batch Snapshot Creation + +```csharp +public sealed class BatchSnapshotService +{ + private readonly ISchedulerLogRepository _logRepository; + private readonly IBatchSnapshotRepository _snapshotRepository; + private readonly IAttestationSigningService? _signingService; + + public async Task CreateSnapshotAsync( + string tenantId, + HlcTimestamp startT, + HlcTimestamp endT, + CancellationToken ct = default) + { + // 1. Select jobs in HLC range + var jobs = await _logRepository.GetByHlcRangeAsync( + tenantId, + startT.ToSortableString(), + endT.ToSortableString(), + ct); + + if (jobs.Count == 0) + { + throw new InvalidOperationException("No jobs in specified HLC range"); + } + + // 2. Get chain head (last link in range) + var headLink = jobs[^1].Link; + + // 3. Create snapshot + var snapshot = new BatchSnapshot + { + BatchId = Guid.NewGuid(), + TenantId = tenantId, + RangeStartT = startT.ToSortableString(), + RangeEndT = endT.ToSortableString(), + HeadLink = headLink, + JobCount = jobs.Count, + CreatedAt = DateTimeOffset.UtcNow + }; + + // 4. Optional: Sign snapshot with DSSE + if (_signingService is not null) + { + var digest = ComputeSnapshotDigest(snapshot, jobs); + var signed = await _signingService.SignAsync(digest, ct); + snapshot = snapshot with + { + SignedBy = signed.KeyId, + Signature = signed.Signature + }; + } + + // 5. Persist + await _snapshotRepository.InsertAsync(snapshot, ct); + + return snapshot; + } +} +``` + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owner | Task Definition | +|---|---------|--------|------------|-------|-----------------| +| 1 | SQC-001 | TODO | HLC lib | Guild | Add StellaOps.HybridLogicalClock reference to Scheduler projects | +| 2 | SQC-002 | TODO | SQC-001 | Guild | Create migration: `scheduler.scheduler_log` table | +| 3 | SQC-003 | TODO | SQC-002 | Guild | Create migration: `scheduler.batch_snapshot` table | +| 4 | SQC-004 | TODO | SQC-002 | Guild | Create migration: `scheduler.chain_heads` table | +| 5 | SQC-005 | TODO | SQC-004 | Guild | Implement `ISchedulerLogRepository` interface | +| 6 | SQC-006 | TODO | SQC-005 | Guild | Implement `PostgresSchedulerLogRepository` | +| 7 | SQC-007 | TODO | SQC-004 | Guild | Implement `IChainHeadRepository` and Postgres implementation | +| 8 | SQC-008 | TODO | SQC-006 | Guild | Implement `SchedulerChainLinking` static class | +| 9 | SQC-009 | TODO | SQC-008 | Guild | Implement `HlcSchedulerEnqueueService` | +| 10 | SQC-010 | TODO | SQC-009 | Guild | Implement `HlcSchedulerDequeueService` | +| 11 | SQC-011 | TODO | SQC-010 | Guild | Update Redis queue adapter to include HLC in message | +| 12 | SQC-012 | TODO | SQC-010 | Guild | Update NATS queue adapter to include HLC in message | +| 13 | SQC-013 | TODO | SQC-006 | Guild | Implement `BatchSnapshotService` | +| 14 | SQC-014 | TODO | SQC-013 | Guild | Add DSSE signing integration for batch snapshots | +| 15 | SQC-015 | TODO | SQC-008 | Guild | Implement chain verification: `VerifyChainIntegrity()` | +| 16 | SQC-016 | TODO | SQC-015 | Guild | Write unit tests: chain linking, HLC ordering | +| 17 | SQC-017 | TODO | SQC-016 | Guild | Write integration tests: enqueue/dequeue with chain | +| 18 | SQC-018 | TODO | SQC-017 | Guild | Write determinism tests: same input -> same chain | +| 19 | SQC-019 | TODO | SQC-018 | Guild | Update existing JobRepository to use HLC ordering optionally | +| 20 | SQC-020 | TODO | SQC-019 | Guild | Feature flag: `SchedulerOptions.EnableHlcOrdering` | +| 21 | SQC-021 | TODO | SQC-020 | Guild | Migration guide: enabling HLC on existing deployments | +| 22 | SQC-022 | TODO | SQC-021 | Guild | Metrics: `scheduler_hlc_enqueues_total`, `scheduler_chain_verifications_total` | + +## Chain Verification + +```csharp +public sealed class SchedulerChainVerifier +{ + public async Task VerifyAsync( + string tenantId, + string? partitionKey, + HlcTimestamp? startT = null, + HlcTimestamp? endT = null, + CancellationToken ct = default) + { + var entries = await _logRepository.GetByHlcRangeAsync( + tenantId, + startT?.ToSortableString(), + endT?.ToSortableString(), + ct); + + byte[]? expectedPrevLink = null; + var issues = new List(); + + foreach (var entry in entries) + { + // Verify prev_link matches expected + if (!ByteArrayEquals(entry.PrevLink, expectedPrevLink)) + { + issues.Add(new ChainVerificationIssue( + entry.JobId, + entry.THlc, + "PrevLinkMismatch", + $"Expected {ToHex(expectedPrevLink)}, got {ToHex(entry.PrevLink)}")); + } + + // Recompute link and verify + var computed = SchedulerChainLinking.ComputeLink( + entry.PrevLink, + entry.JobId, + HlcTimestamp.Parse(entry.THlc), + entry.PayloadHash); + + if (!ByteArrayEquals(entry.Link, computed)) + { + issues.Add(new ChainVerificationIssue( + entry.JobId, + entry.THlc, + "LinkMismatch", + $"Stored link doesn't match computed")); + } + + expectedPrevLink = entry.Link; + } + + return new ChainVerificationResult( + IsValid: issues.Count == 0, + EntriesChecked: entries.Count, + Issues: issues); + } +} +``` + +## Backward Compatibility + +### Feature Flag + +```csharp +public sealed class SchedulerOptions +{ + /// + /// Enable HLC-based ordering with chain linking. + /// When false, uses legacy (priority, created_at) ordering. + /// + public bool EnableHlcOrdering { get; set; } = false; + + /// + /// When true, writes to both legacy and HLC tables during migration. + /// + public bool DualWriteMode { get; set; } = false; +} +``` + +### Migration Path + +1. **Phase 1:** Deploy with `DualWriteMode = true` - writes to both tables +2. **Phase 2:** Backfill `scheduler_log` from existing `scheduler.jobs` +3. **Phase 3:** Enable `EnableHlcOrdering = true` for reads +4. **Phase 4:** Disable `DualWriteMode`, deprecate legacy ordering + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Separate `scheduler_log` table | Avoid schema changes to existing `jobs` table | +| Store HLC as TEXT | Human-readable, sortable, avoids custom types | +| SHA-256 for chain links | Consistent with existing hash usage; pluggable | +| Optional DSSE signing | Not all deployments need attestation | + +| Risk | Mitigation | +|------|------------| +| Performance regression | Benchmark; index optimization; optional feature | +| Chain corruption | Verification function; alerts on mismatch | +| Migration complexity | Dual-write mode; gradual rollout | + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +## Next Checkpoints + +- 2026-01-09: SQC-001 to SQC-008 complete (schema + core) +- 2026-01-10: SQC-009 to SQC-014 complete (services) +- 2026-01-11: SQC-015 to SQC-022 complete (verification, tests, docs) diff --git a/docs/implplan/SPRINT_20260105_002_003_FACET_perfacet_quotas.md b/docs/implplan/SPRINT_20260105_002_003_FACET_perfacet_quotas.md new file mode 100644 index 000000000..3d510462f --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_003_FACET_perfacet_quotas.md @@ -0,0 +1,701 @@ +# Sprint 20260105_002_003_FACET - Per-Facet Drift Quotas + +## Topic & Scope + +Implement per-facet drift quota enforcement that tracks changes against sealed baselines and applies configurable thresholds with WARN/BLOCK/RequireVex actions. This sprint extends the existing `FnDriftCalculator` to support facet-level granularity. + +**Advisory Reference:** Product advisory on facet sealing - "Track drift" and "Quotas & actions" sections. + +**Key Insight:** Different facets should have different drift tolerances. OS package updates are expected during patching, but binary changes outside of known patches are suspicious. Per-facet quotas enable nuanced enforcement. + +**Working directory:** `src/__Libraries/StellaOps.Facet/`, `src/Policy/__Libraries/StellaOps.Policy/Gates/` + +**Evidence:** `IFacetQuotaEnforcer` with per-facet drift tracking, quota breach actions integrated into policy gates, auto-VEX draft generation for authorized drift. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| SPRINT_20260105_002_002_FACET (models) | Sprint | Required | +| FnDriftCalculator | Internal | Available | +| BudgetConstraintEnforcer | Internal | Available | +| DeltaSigVexEmitter | Internal | Available | +| FacetSeal model | Sprint 002 | Required | + +**Parallel Execution:** QTA-001 through QTA-008 (drift engine) can proceed independently. QTA-009 through QTA-015 (enforcement) depend on drift engine. QTA-016 through QTA-020 (auto-VEX) can proceed in parallel with enforcement. + +--- + +## Documentation Prerequisites + +- SPRINT_20260105_002_002_FACET models +- `src/Scanner/__Libraries/StellaOps.Scanner.Storage/Services/FnDriftCalculator.cs` +- `src/Policy/__Libraries/StellaOps.Policy/Gates/BudgetConstraintEnforcer.cs` +- `src/Scanner/__Libraries/StellaOps.Scanner.Evidence/DeltaSigVexEmitter.cs` + +--- + +## Problem Analysis + +### Current State + +StellaOps currently: +- Tracks aggregate FN-Drift with cause attribution (Feed, Rule, Lattice, Reachability, Engine) +- Has budget enforcement with Risk Points (RP) and gate levels +- Generates auto-VEX from delta signature detection +- No per-facet drift tracking +- No facet-specific quota configuration +- No quota-based BLOCK actions + +**Gaps:** +1. `FnDriftCalculator` operates at artifact level, not facet level +2. No `FacetDriftEngine` to compare current vs sealed baseline +3. No quota enforcement per facet +4. No facet-aware auto-VEX generation + +### Target Capabilities + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ Per-Facet Quota Enforcement │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ FacetDriftEngine │ │ +│ │ │ │ +│ │ Input: Current Image + Sealed Baseline │ │ +│ │ │ │ +│ │ For each facet: │ │ +│ │ 1. Extract current files via IFacetExtractor │ │ +│ │ 2. Load baseline FacetEntry from FacetSeal │ │ +│ │ 3. Compute diff: added, removed, modified │ │ +│ │ 4. Calculate drift score and churn % │ │ +│ │ 5. Evaluate quota: MaxChurnPercent, MaxChangedFiles │ │ +│ │ 6. Determine verdict: OK / Warning / Block / RequiresVex │ │ +│ │ │ │ +│ │ Output: FacetDriftReport with per-facet FacetDrift entries │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ Quota Configuration Example │ │ +│ │ │ │ +│ │ facet_quotas: │ │ +│ │ os-packages-dpkg: │ │ +│ │ max_churn_percent: 15 │ │ +│ │ max_changed_files: 100 │ │ +│ │ action: warn │ │ +│ │ allowlist: │ │ +│ │ - "/var/lib/dpkg/status" # Expected to change │ │ +│ │ │ │ +│ │ binaries-usr: │ │ +│ │ max_churn_percent: 5 │ │ +│ │ max_changed_files: 10 │ │ +│ │ action: block # Binaries shouldn't change unexpectedly │ │ +│ │ │ │ +│ │ lang-deps-npm: │ │ +│ │ max_churn_percent: 25 │ │ +│ │ max_changed_files: 200 │ │ +│ │ action: require_vex │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ Auto-VEX from Drift │ │ +│ │ │ │ +│ │ When drift is detected and action = require_vex: │ │ +│ │ 1. Generate draft VEX statement with status "under_investigation" │ │ +│ │ 2. Include drift context: facet, files changed, churn % │ │ +│ │ 3. Queue for human review in VEX workflow │ │ +│ │ 4. If approved, drift is "authorized" and quota resets │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Architecture Design + +### Facet Drift Engine + +```csharp +// src/__Libraries/StellaOps.Facet/Drift/IFacetDriftEngine.cs +namespace StellaOps.Facet.Drift; + +/// +/// Computes drift between current image state and sealed baseline. +/// +public interface IFacetDriftEngine +{ + /// + /// Compute drift for all facets in a seal. + /// + Task ComputeDriftAsync( + FacetSeal baseline, + IImageFileSystem currentImage, + FacetDriftOptions? options = null, + CancellationToken ct = default); + + /// + /// Compute drift for a single facet. + /// + Task ComputeFacetDriftAsync( + FacetEntry baselineEntry, + IImageFileSystem currentImage, + FacetQuota? quota = null, + CancellationToken ct = default); +} + +public sealed record FacetDriftReport +{ + /// + /// Image being analyzed. + /// + public required string ImageDigest { get; init; } + + /// + /// Baseline seal used for comparison. + /// + public required string BaselineSealId { get; init; } + + /// + /// When drift analysis was performed. + /// + public required DateTimeOffset AnalyzedAt { get; init; } + + /// + /// Per-facet drift results. + /// + public required ImmutableArray FacetDrifts { get; init; } + + /// + /// Overall verdict (worst of all facets). + /// + public required QuotaVerdict OverallVerdict { get; init; } + + /// + /// Facets that exceeded quota. + /// + public ImmutableArray QuotaBreaches => + FacetDrifts.Where(d => d.QuotaVerdict != QuotaVerdict.Ok) + .Select(d => d.FacetId) + .ToImmutableArray(); + + /// + /// Total files changed across all facets. + /// + public int TotalChangedFiles => + FacetDrifts.Sum(d => d.Added.Length + d.Removed.Length + d.Modified.Length); +} + +public sealed record FacetDriftOptions +{ + /// + /// Custom quota overrides per facet. + /// + public ImmutableDictionary? QuotaOverrides { get; init; } + + /// + /// Skip drift computation for these facets. + /// + public ImmutableArray SkipFacets { get; init; } = []; + + /// + /// Include detailed file lists (slower but useful for debugging). + /// + public bool IncludeFileDetails { get; init; } = true; +} +``` + +### Facet Drift Engine Implementation + +```csharp +// src/__Libraries/StellaOps.Facet/Drift/FacetDriftEngine.cs +namespace StellaOps.Facet.Drift; + +internal sealed class FacetDriftEngine : IFacetDriftEngine +{ + private readonly IFacetExtractor _extractor; + private readonly FacetMerkleTree _merkleTree; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public FacetDriftEngine( + IFacetExtractor extractor, + FacetMerkleTree merkleTree, + TimeProvider? timeProvider = null, + ILogger? logger = null) + { + _extractor = extractor; + _merkleTree = merkleTree; + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? NullLogger.Instance; + } + + public async Task ComputeDriftAsync( + FacetSeal baseline, + IImageFileSystem currentImage, + FacetDriftOptions? options = null, + CancellationToken ct = default) + { + options ??= new FacetDriftOptions(); + var drifts = new List(); + var worstVerdict = QuotaVerdict.Ok; + + foreach (var facetEntry in baseline.Facets) + { + ct.ThrowIfCancellationRequested(); + + if (options.SkipFacets.Contains(facetEntry.FacetId)) + { + _logger.LogDebug("Skipping facet {FacetId} per options", facetEntry.FacetId); + continue; + } + + // Get quota (override or from baseline) + var quota = options.QuotaOverrides?.GetValueOrDefault(facetEntry.FacetId) + ?? baseline.Quotas?.GetValueOrDefault(facetEntry.FacetId); + + var drift = await ComputeFacetDriftAsync(facetEntry, currentImage, quota, ct) + .ConfigureAwait(false); + + drifts.Add(drift); + + if (drift.QuotaVerdict > worstVerdict) + { + worstVerdict = drift.QuotaVerdict; + } + } + + return new FacetDriftReport + { + ImageDigest = baseline.ImageDigest, + BaselineSealId = baseline.CombinedMerkleRoot, + AnalyzedAt = _timeProvider.GetUtcNow(), + FacetDrifts = [.. drifts], + OverallVerdict = worstVerdict + }; + } + + public async Task ComputeFacetDriftAsync( + FacetEntry baselineEntry, + IImageFileSystem currentImage, + FacetQuota? quota = null, + CancellationToken ct = default) + { + // Get facet definition + var facet = BuiltInFacets.GetById(baselineEntry.FacetId) + ?? throw new InvalidOperationException($"Unknown facet: {baselineEntry.FacetId}"); + + // Extract current files + var extraction = await _extractor.ExtractAsync(facet, currentImage, ct: ct) + .ConfigureAwait(false); + + // Build lookup maps + var baselineFiles = baselineEntry.Files?.ToDictionary(f => f.Path) ?? new(); + var currentFiles = extraction.Files.ToDictionary(f => f.Path); + + // Compute diff + var added = new List(); + var removed = new List(); + var modified = new List(); + + // Files in current but not baseline = added + foreach (var file in extraction.Files) + { + if (!baselineFiles.ContainsKey(file.Path)) + { + added.Add(file); + } + } + + // Files in baseline + foreach (var (path, baselineFile) in baselineFiles) + { + if (!currentFiles.TryGetValue(path, out var currentFile)) + { + // In baseline but not current = removed + removed.Add(baselineFile); + } + else if (baselineFile.Digest != currentFile.Digest) + { + // In both but different = modified + modified.Add(new FacetFileModification( + path, + baselineFile.Digest, + currentFile.Digest, + baselineFile.SizeBytes, + currentFile.SizeBytes)); + } + } + + // Apply allowlist filtering + if (quota?.AllowlistGlobs.Length > 0) + { + var allowedPaths = new HashSet(); + foreach (var glob in quota.AllowlistGlobs) + { + // Simple glob matching (would use DotNet.Glob in production) + allowedPaths.UnionWith(FilterByGlob( + added.Select(f => f.Path) + .Concat(removed.Select(f => f.Path)) + .Concat(modified.Select(m => m.Path)), + glob)); + } + + added = added.Where(f => !allowedPaths.Contains(f.Path)).ToList(); + removed = removed.Where(f => !allowedPaths.Contains(f.Path)).ToList(); + modified = modified.Where(m => !allowedPaths.Contains(m.Path)).ToList(); + } + + // Calculate metrics + var totalChanges = added.Count + removed.Count + modified.Count; + var churnPercent = baselineEntry.FileCount > 0 + ? totalChanges / (decimal)baselineEntry.FileCount * 100 + : 0; + + var driftScore = ComputeDriftScore(added.Count, removed.Count, modified.Count, churnPercent); + + // Evaluate quota + var verdict = EvaluateQuota(quota, churnPercent, totalChanges); + + return new FacetDrift + { + FacetId = baselineEntry.FacetId, + Added = [.. added], + Removed = [.. removed], + Modified = [.. modified], + DriftScore = driftScore, + QuotaVerdict = verdict, + BaselineFileCount = baselineEntry.FileCount + }; + } + + private static decimal ComputeDriftScore(int added, int removed, int modified, decimal churnPercent) + { + // Weighted score: removals and modifications are more significant than additions + const decimal addWeight = 1.0m; + const decimal removeWeight = 2.0m; + const decimal modifyWeight = 1.5m; + + var weightedChanges = added * addWeight + removed * removeWeight + modified * modifyWeight; + + // Normalize to 0-100 scale based on churn + return Math.Min(100, churnPercent + weightedChanges / 10); + } + + private static QuotaVerdict EvaluateQuota(FacetQuota? quota, decimal churnPercent, int totalChanges) + { + if (quota is null) + { + return QuotaVerdict.Ok; // No quota = no enforcement + } + + var breached = churnPercent > quota.MaxChurnPercent || totalChanges > quota.MaxChangedFiles; + + if (!breached) + { + return QuotaVerdict.Ok; + } + + return quota.Action switch + { + QuotaExceededAction.Warn => QuotaVerdict.Warning, + QuotaExceededAction.Block => QuotaVerdict.Blocked, + QuotaExceededAction.RequireVex => QuotaVerdict.RequiresVex, + _ => QuotaVerdict.Warning + }; + } + + private static IEnumerable FilterByGlob(IEnumerable paths, string glob) + { + // Simplified glob matching - use DotNet.Glob in production + var pattern = "^" + Regex.Escape(glob) + .Replace("\\*\\*", ".*") + .Replace("\\*", "[^/]*") + "$"; + + var regex = new Regex(pattern, RegexOptions.Compiled); + return paths.Where(p => regex.IsMatch(p)); + } +} +``` + +### Quota Enforcer Gate + +```csharp +// src/Policy/__Libraries/StellaOps.Policy/Gates/FacetQuotaGate.cs +namespace StellaOps.Policy.Gates; + +/// +/// Policy gate that enforces facet drift quotas. +/// +public sealed class FacetQuotaGate : IGateEvaluator +{ + private readonly IFacetDriftEngine _driftEngine; + private readonly IFacetSealStore _sealStore; + private readonly ILogger _logger; + + public string GateId => "facet-quota"; + public string DisplayName => "Facet Drift Quota"; + public int Priority => 50; // After evidence freshness, before budget + + public FacetQuotaGate( + IFacetDriftEngine driftEngine, + IFacetSealStore sealStore, + ILogger? logger = null) + { + _driftEngine = driftEngine; + _sealStore = sealStore; + _logger = logger ?? NullLogger.Instance; + } + + public async Task EvaluateAsync( + GateContext context, + CancellationToken ct = default) + { + // Check if facet quota enforcement is enabled + if (!context.PolicyOptions.FacetQuotaEnabled) + { + return GateResult.Pass("Facet quota enforcement disabled"); + } + + // Load baseline seal + var baseline = await _sealStore.GetLatestSealAsync(context.ImageDigest, ct) + .ConfigureAwait(false); + + if (baseline is null) + { + _logger.LogWarning("No baseline seal found for {Image}, skipping quota check", + context.ImageDigest); + return GateResult.Pass("No baseline seal - quota check skipped"); + } + + // Compute drift + var driftReport = await _driftEngine.ComputeDriftAsync( + baseline, + context.ImageFileSystem, + ct: ct).ConfigureAwait(false); + + // Evaluate result + return driftReport.OverallVerdict switch + { + QuotaVerdict.Ok => GateResult.Pass( + $"Facet quotas OK: {driftReport.TotalChangedFiles} files changed"), + + QuotaVerdict.Warning => GateResult.Warn( + $"Facet quota warning: {FormatBreaches(driftReport)}", + new GateWarning("facet.quota.warning", driftReport.QuotaBreaches)), + + QuotaVerdict.Blocked => GateResult.Block( + $"Facet quota BLOCKED: {FormatBreaches(driftReport)}", + BlockReason.QuotaExceeded), + + QuotaVerdict.RequiresVex => GateResult.RequiresAction( + $"Facet drift requires VEX: {FormatBreaches(driftReport)}", + RequiredAction.SubmitVex, + GenerateVexContext(driftReport)), + + _ => GateResult.Pass("Unknown verdict - defaulting to pass") + }; + } + + private static string FormatBreaches(FacetDriftReport report) + { + return string.Join(", ", report.FacetDrifts + .Where(d => d.QuotaVerdict != QuotaVerdict.Ok) + .Select(d => $"{d.FacetId}({d.ChurnPercent:F1}%)")); + } + + private static VexContext GenerateVexContext(FacetDriftReport report) + { + return new VexContext + { + ContextType = "facet-drift", + ArtifactDigest = report.ImageDigest, + FacetBreaches = report.QuotaBreaches.ToList(), + TotalChangedFiles = report.TotalChangedFiles, + AnalyzedAt = report.AnalyzedAt + }; + } +} +``` + +### Auto-VEX from Drift + +```csharp +// src/__Libraries/StellaOps.Facet/Vex/FacetDriftVexEmitter.cs +namespace StellaOps.Facet.Vex; + +/// +/// Generates draft VEX statements from facet drift requiring authorization. +/// +public sealed class FacetDriftVexEmitter +{ + private readonly IVexDraftStore _draftStore; + private readonly TimeProvider _timeProvider; + private readonly IGuidProvider _guidProvider; + + public FacetDriftVexEmitter( + IVexDraftStore draftStore, + TimeProvider? timeProvider = null, + IGuidProvider? guidProvider = null) + { + _draftStore = draftStore; + _timeProvider = timeProvider ?? TimeProvider.System; + _guidProvider = guidProvider ?? SystemGuidProvider.Instance; + } + + /// + /// Generate draft VEX statements for facets requiring authorization. + /// + public async Task> GenerateDraftsAsync( + FacetDriftReport report, + CancellationToken ct = default) + { + var drafts = new List(); + + foreach (var drift in report.FacetDrifts.Where(d => d.QuotaVerdict == QuotaVerdict.RequiresVex)) + { + var draft = CreateDraft(report, drift); + await _draftStore.SaveAsync(draft, ct).ConfigureAwait(false); + drafts.Add(draft); + } + + return [.. drafts]; + } + + private VexDraft CreateDraft(FacetDriftReport report, FacetDrift drift) + { + var now = _timeProvider.GetUtcNow(); + + return new VexDraft + { + DraftId = $"vex-draft:{_guidProvider.NewGuid()}", + Status = VexStatus.UnderInvestigation, + Category = "facet-drift-authorization", + CreatedAt = now, + ArtifactDigest = report.ImageDigest, + FacetId = drift.FacetId, + Justification = GenerateJustification(drift), + Context = new VexDraftContext + { + BaselineSealId = report.BaselineSealId, + ChurnPercent = drift.ChurnPercent, + FilesAdded = drift.Added.Length, + FilesRemoved = drift.Removed.Length, + FilesModified = drift.Modified.Length, + SampleChanges = GetSampleChanges(drift, maxSamples: 10) + }, + RequiresReview = true, + ReviewDeadline = now.AddDays(7) // 7-day SLA for drift review + }; + } + + private static string GenerateJustification(FacetDrift drift) + { + var sb = new StringBuilder(); + sb.AppendLine($"Facet '{drift.FacetId}' exceeded drift quota."); + sb.AppendLine($"Churn: {drift.ChurnPercent:F2}% ({drift.Added.Length} added, {drift.Removed.Length} removed, {drift.Modified.Length} modified)"); + sb.AppendLine(); + sb.AppendLine("Review required to authorize this drift. Possible reasons:"); + sb.AppendLine("- Planned security patch deployment"); + sb.AppendLine("- Dependency update"); + sb.AppendLine("- Build reproducibility variance"); + sb.AppendLine("- Unauthorized modification (investigate)"); + return sb.ToString(); + } + + private static ImmutableArray GetSampleChanges(FacetDrift drift, int maxSamples) + { + var samples = new List(); + + foreach (var added in drift.Added.Take(maxSamples / 3)) + samples.Add($"+ {added.Path}"); + + foreach (var removed in drift.Removed.Take(maxSamples / 3)) + samples.Add($"- {removed.Path}"); + + foreach (var modified in drift.Modified.Take(maxSamples / 3)) + samples.Add($"~ {modified.Path}"); + + return [.. samples]; + } +} +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| **Drift Engine** | +| 1 | QTA-001 | TODO | FCT models | Facet Guild | Define `IFacetDriftEngine` interface | +| 2 | QTA-002 | TODO | QTA-001 | Facet Guild | Define `FacetDriftReport` model | +| 3 | QTA-003 | TODO | QTA-002 | Facet Guild | Implement file diff computation (added/removed/modified) | +| 4 | QTA-004 | TODO | QTA-003 | Facet Guild | Implement allowlist glob filtering | +| 5 | QTA-005 | TODO | QTA-004 | Facet Guild | Implement drift score calculation | +| 6 | QTA-006 | TODO | QTA-005 | Facet Guild | Implement quota evaluation logic | +| 7 | QTA-007 | TODO | QTA-006 | Facet Guild | Unit tests: Drift computation with fixtures | +| 8 | QTA-008 | TODO | QTA-007 | Facet Guild | Unit tests: Quota evaluation edge cases | +| **Quota Enforcement** | +| 9 | QTA-009 | TODO | QTA-006 | Policy Guild | Create `FacetQuotaGate` class | +| 10 | QTA-010 | TODO | QTA-009 | Policy Guild | Integrate with `IGateEvaluator` pipeline | +| 11 | QTA-011 | TODO | QTA-010 | Policy Guild | Add `FacetQuotaEnabled` to policy options | +| 12 | QTA-012 | TODO | QTA-011 | Policy Guild | Create `IFacetSealStore` for baseline lookups | +| 13 | QTA-013 | TODO | QTA-012 | Policy Guild | Implement Postgres storage for facet seals | +| 14 | QTA-014 | TODO | QTA-013 | Policy Guild | Unit tests: Gate evaluation scenarios | +| 15 | QTA-015 | TODO | QTA-014 | Policy Guild | Integration tests: Full gate pipeline | +| **Auto-VEX Generation** | +| 16 | QTA-016 | TODO | QTA-006 | VEX Guild | Create `FacetDriftVexEmitter` class | +| 17 | QTA-017 | TODO | QTA-016 | VEX Guild | Define `VexDraft` and `VexDraftContext` models | +| 18 | QTA-018 | TODO | QTA-017 | VEX Guild | Implement draft storage and retrieval | +| 19 | QTA-019 | TODO | QTA-018 | VEX Guild | Wire into Excititor VEX workflow | +| 20 | QTA-020 | TODO | QTA-019 | VEX Guild | Unit tests: Draft generation and justification | +| **Configuration & Documentation** | +| 21 | QTA-021 | TODO | QTA-015 | Ops Guild | Create facet quota YAML schema | +| 22 | QTA-022 | TODO | QTA-021 | Ops Guild | Add default quota profiles (strict, moderate, permissive) | +| 23 | QTA-023 | TODO | QTA-022 | Docs Guild | Document quota configuration in ops guide | +| 24 | QTA-024 | TODO | QTA-023 | QA Guild | E2E test: Quota breach → VEX draft → approval | + +--- + +## Success Metrics + +| Metric | Before | After | Target | +|--------|--------|-------|--------| +| Per-facet drift tracking | No | Yes | All facets | +| Quota enforcement per facet | No | Yes | Configurable | +| BLOCK action on quota breach | No | Yes | Functional | +| Auto-VEX draft from drift | No | Yes | Queued for review | +| 50% fewer noisy regressions | Baseline | Measured | 50% reduction | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| Drift computation adds latency | Trade-off | Make optional, cache baselines | +| Allowlist globs may be too broad | Risk | Document best practices, provide linting | +| VEX draft SLA enforcement | Decision | 7-day default, configurable per tenant | +| Storage growth from drift history | Risk | Retention policy, aggregate old data | + +--- + +## Next Checkpoints + +- QTA-001 through QTA-008 (drift engine) target completion +- QTA-009 through QTA-015 (enforcement) target completion +- QTA-016 through QTA-020 (auto-VEX) target completion +- QTA-024 (E2E) sprint completion gate diff --git a/docs/implplan/SPRINT_20260105_002_003_ROUTER_hlc_offline_merge.md b/docs/implplan/SPRINT_20260105_002_003_ROUTER_hlc_offline_merge.md new file mode 100644 index 000000000..2d852a43d --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_003_ROUTER_hlc_offline_merge.md @@ -0,0 +1,444 @@ +# Sprint 20260105_002_003_ROUTER - HLC: Offline Merge Protocol + +## Topic & Scope + +Implement HLC-based deterministic merge protocol for offline/air-gap scenarios. When disconnected nodes sync, jobs must merge by HLC order key to maintain global ordering without conflicts. + +- **Working directory:** `src/Router/`, `src/AirGap/` +- **Evidence:** Merge algorithm implementation, conflict resolution tests, air-gap sync integration + +## Problem Statement + +Current air-gap handling: +- Staleness validation gates job scheduling +- No deterministic merge protocol for offline-enqueued jobs +- Wall-clock based ordering causes drift on reconnection + +Advisory requires: +- Enqueue locally with HLC rules and chain links +- On sync, merge by order key `(T_hlc, PartitionKey?, JobId)` +- Merges are conflict-free because keys are deterministic + +## Dependencies & Concurrency + +- **Depends on:** SPRINT_20260105_002_002_SCHEDULER (HLC queue chain) +- **Blocks:** SPRINT_20260105_002_004_BE (integration tests) +- **Parallel safe:** Router/AirGap changes isolated from other modules + +## Documentation Prerequisites + +- docs/modules/router/architecture.md +- docs/airgap/OFFLINE_KIT.md +- src/AirGap/AGENTS.md +- Product Advisory: offline + replay section + +## Technical Design + +### Offline HLC Persistence + +When operating in air-gap/offline mode, each node maintains its own HLC state: + +```csharp +public sealed class OfflineHlcManager +{ + private readonly IHybridLogicalClock _hlc; + private readonly IOfflineJobLogStore _jobLogStore; + private readonly string _nodeId; + + public OfflineHlcManager( + IHybridLogicalClock hlc, + IOfflineJobLogStore jobLogStore, + string nodeId) + { + _hlc = hlc; + _jobLogStore = jobLogStore; + _nodeId = nodeId; + } + + /// + /// Enqueue job locally while offline. Maintains local chain. + /// + public async Task EnqueueOfflineAsync( + SchedulerJobPayload payload, + CancellationToken ct = default) + { + var tHlc = _hlc.Tick(); + var jobId = ComputeDeterministicJobId(payload); + var payloadHash = SchedulerChainLinking.ComputePayloadHash(payload); + + var prevLink = await _jobLogStore.GetLastLinkAsync(_nodeId, ct); + var link = SchedulerChainLinking.ComputeLink(prevLink, jobId, tHlc, payloadHash); + + var entry = new OfflineJobLogEntry + { + NodeId = _nodeId, + THlc = tHlc, + JobId = jobId, + Payload = payload, + PayloadHash = payloadHash, + PrevLink = prevLink, + Link = link, + EnqueuedAt = DateTimeOffset.UtcNow + }; + + await _jobLogStore.AppendAsync(entry, ct); + + return new OfflineEnqueueResult(tHlc, jobId, link, _nodeId); + } +} +``` + +### Merge Algorithm + +The merge algorithm combines job logs from multiple nodes while preserving HLC ordering: + +```csharp +public sealed class HlcMergeService +{ + /// + /// Merge job logs from multiple offline nodes into unified, HLC-ordered stream. + /// + public async Task MergeAsync( + IReadOnlyList nodeLogs, + CancellationToken ct = default) + { + // 1. Collect all entries from all nodes + var allEntries = nodeLogs + .SelectMany(log => log.Entries.Select(e => (log.NodeId, Entry: e))) + .ToList(); + + // 2. Sort by HLC total order: (PhysicalTime, LogicalCounter, NodeId, JobId) + var sorted = allEntries + .OrderBy(x => x.Entry.THlc.PhysicalTime) + .ThenBy(x => x.Entry.THlc.LogicalCounter) + .ThenBy(x => x.Entry.THlc.NodeId, StringComparer.Ordinal) + .ThenBy(x => x.Entry.JobId) + .ToList(); + + // 3. Detect duplicates (same JobId = same deterministic payload) + var seen = new HashSet(); + var deduplicated = new List(); + var duplicates = new List(); + + foreach (var (nodeId, entry) in sorted) + { + if (seen.Contains(entry.JobId)) + { + duplicates.Add(new DuplicateEntry(entry.JobId, nodeId, entry.THlc)); + continue; + } + + seen.Add(entry.JobId); + deduplicated.Add(new MergedJobEntry + { + SourceNodeId = nodeId, + THlc = entry.THlc, + JobId = entry.JobId, + Payload = entry.Payload, + PayloadHash = entry.PayloadHash, + OriginalLink = entry.Link + }); + } + + // 4. Recompute unified chain + byte[]? prevLink = null; + foreach (var entry in deduplicated) + { + entry.MergedLink = SchedulerChainLinking.ComputeLink( + prevLink, + entry.JobId, + entry.THlc, + entry.PayloadHash); + prevLink = entry.MergedLink; + } + + return new MergeResult + { + MergedEntries = deduplicated, + Duplicates = duplicates, + MergedChainHead = prevLink, + SourceNodes = nodeLogs.Select(l => l.NodeId).ToList() + }; + } +} +``` + +### Sync Protocol + +```csharp +public sealed class AirGapSyncService +{ + private readonly IHlcMergeService _mergeService; + private readonly ISchedulerLogRepository _schedulerLogRepo; + private readonly IHybridLogicalClock _hlc; + + /// + /// Sync offline jobs from air-gap bundle to central scheduler. + /// + public async Task SyncFromBundleAsync( + AirGapBundle bundle, + CancellationToken ct = default) + { + var nodeLogs = bundle.JobLogs; + + // 1. Merge all offline logs + var merged = await _mergeService.MergeAsync(nodeLogs, ct); + + // 2. Get current scheduler chain head + var currentHead = await _schedulerLogRepo.GetChainHeadAsync( + bundle.TenantId, + ct); + + // 3. For each merged entry, update HLC clock (receive) + // This ensures central clock advances past all offline timestamps + foreach (var entry in merged.MergedEntries) + { + _hlc.Receive(entry.THlc); + } + + // 4. Append merged entries to scheduler log + // Chain links recomputed to extend from current head + byte[]? prevLink = currentHead?.Link; + var appended = new List(); + + foreach (var entry in merged.MergedEntries) + { + // Check if job already exists (idempotency) + var existing = await _schedulerLogRepo.GetByJobIdAsync( + bundle.TenantId, + entry.JobId, + ct); + + if (existing is not null) + { + continue; // Already synced + } + + var newLink = SchedulerChainLinking.ComputeLink( + prevLink, + entry.JobId, + entry.THlc, + entry.PayloadHash); + + var logEntry = new SchedulerLogEntry + { + TenantId = bundle.TenantId, + THlc = entry.THlc.ToSortableString(), + PartitionKey = entry.Payload.PartitionKey, + JobId = entry.JobId, + PayloadHash = entry.PayloadHash, + PrevLink = prevLink, + Link = newLink, + SourceNodeId = entry.SourceNodeId, + SyncedFromBundle = bundle.BundleId + }; + + await _schedulerLogRepo.InsertAsync(logEntry, ct); + appended.Add(logEntry); + prevLink = newLink; + } + + return new SyncResult + { + BundleId = bundle.BundleId, + TotalInBundle = merged.MergedEntries.Count, + Appended = appended.Count, + Duplicates = merged.Duplicates.Count, + NewChainHead = prevLink + }; + } +} +``` + +### Air-Gap Bundle Format + +```csharp +public sealed record AirGapBundle +{ + public required Guid BundleId { get; init; } + public required string TenantId { get; init; } + public required DateTimeOffset CreatedAt { get; init; } + public required string CreatedByNodeId { get; init; } + + /// Job logs from each offline node. + public required IReadOnlyList JobLogs { get; init; } + + /// Bundle manifest digest for integrity. + public required string ManifestDigest { get; init; } + + /// Optional DSSE signature over manifest. + public string? Signature { get; init; } +} + +public sealed record NodeJobLog +{ + public required string NodeId { get; init; } + public required HlcTimestamp LastHlc { get; init; } + public required byte[] ChainHead { get; init; } + public required IReadOnlyList Entries { get; init; } +} +``` + +### Conflict Resolution + +HLC ensures conflicts are rare, but when they occur: + +```csharp +public sealed class ConflictResolver +{ + /// + /// Resolve conflicts when same JobId has different payloads. + /// This should NOT happen with deterministic JobId computation. + /// + public ConflictResolution Resolve( + Guid jobId, + IReadOnlyList<(string NodeId, OfflineJobLogEntry Entry)> conflicting) + { + // Verify payloads are actually different + var uniquePayloads = conflicting + .Select(c => Convert.ToHexString(c.Entry.PayloadHash)) + .Distinct() + .ToList(); + + if (uniquePayloads.Count == 1) + { + // Same payload, different HLC - not a real conflict + // Take earliest HLC (preserves causality) + var earliest = conflicting + .OrderBy(c => c.Entry.THlc) + .First(); + + return new ConflictResolution + { + Type = ConflictType.DuplicateTimestamp, + Resolution = ResolutionStrategy.TakeEarliest, + SelectedEntry = earliest.Entry, + DroppedEntries = conflicting + .Where(c => c.Entry != earliest.Entry) + .Select(c => c.Entry) + .ToList() + }; + } + + // Actual conflict: same JobId, different payloads + // This indicates a bug in deterministic ID computation + return new ConflictResolution + { + Type = ConflictType.PayloadMismatch, + Resolution = ResolutionStrategy.Error, + Error = $"JobId {jobId} has conflicting payloads from nodes: " + + string.Join(", ", conflicting.Select(c => c.NodeId)) + }; + } +} +``` + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owner | Task Definition | +|---|---------|--------|------------|-------|-----------------| +| 1 | OMP-001 | TODO | SQC lib | Guild | Create `StellaOps.AirGap.Sync` library project | +| 2 | OMP-002 | TODO | OMP-001 | Guild | Implement `OfflineHlcManager` for local offline enqueue | +| 3 | OMP-003 | TODO | OMP-002 | Guild | Implement `IOfflineJobLogStore` and file-based store | +| 4 | OMP-004 | TODO | OMP-003 | Guild | Implement `HlcMergeService` with total order merge | +| 5 | OMP-005 | TODO | OMP-004 | Guild | Implement `ConflictResolver` for edge cases | +| 6 | OMP-006 | TODO | OMP-005 | Guild | Implement `AirGapSyncService` for bundle import | +| 7 | OMP-007 | TODO | OMP-006 | Guild | Define `AirGapBundle` format (JSON schema) | +| 8 | OMP-008 | TODO | OMP-007 | Guild | Implement bundle export: `AirGapBundleExporter` | +| 9 | OMP-009 | TODO | OMP-008 | Guild | Implement bundle import: `AirGapBundleImporter` | +| 10 | OMP-010 | TODO | OMP-009 | Guild | Add DSSE signing for bundle integrity | +| 11 | OMP-011 | TODO | OMP-006 | Guild | Integrate with Router transport layer | +| 12 | OMP-012 | TODO | OMP-011 | Guild | Update `stella airgap export` CLI command | +| 13 | OMP-013 | TODO | OMP-012 | Guild | Update `stella airgap import` CLI command | +| 14 | OMP-014 | TODO | OMP-004 | Guild | Write unit tests: merge algorithm correctness | +| 15 | OMP-015 | TODO | OMP-014 | Guild | Write unit tests: duplicate detection | +| 16 | OMP-016 | TODO | OMP-015 | Guild | Write unit tests: conflict resolution | +| 17 | OMP-017 | TODO | OMP-016 | Guild | Write integration tests: offline -> online sync | +| 18 | OMP-018 | TODO | OMP-017 | Guild | Write integration tests: multi-node merge | +| 19 | OMP-019 | TODO | OMP-018 | Guild | Write determinism tests: same bundles -> same result | +| 20 | OMP-020 | TODO | OMP-019 | Guild | Metrics: `airgap_sync_total`, `airgap_merge_conflicts_total` | +| 21 | OMP-021 | TODO | OMP-020 | Guild | Documentation: offline operations guide | + +## Test Scenarios + +### Scenario 1: Simple Two-Node Merge + +``` +Node A (offline): Node B (offline): + Job1 @ T=100 Job2 @ T=101 + Job3 @ T=102 Job4 @ T=103 + +After merge (HLC order): + Job1 @ T=100 (from A) + Job2 @ T=101 (from B) + Job3 @ T=102 (from A) + Job4 @ T=103 (from B) +``` + +### Scenario 2: Same Payload, Different Nodes + +``` +Node A: Job(payload=X) @ T=100 -> JobId=abc123 +Node B: Job(payload=X) @ T=105 -> JobId=abc123 + +Result: Single entry with T=100 (earliest), duplicate at T=105 dropped +``` + +### Scenario 3: Clock Skew During Offline + +``` +Node A (clock +5min): Job1 @ T=300 (actually T=0) +Node B (clock correct): Job2 @ T=100 + +After merge with HLC receive(): + Central clock advances to max(local, 300) + Order: Job2 @ T=100, Job1 @ T=300 (logical order preserved) +``` + +## Metrics & Observability + +``` +# Counters +airgap_bundles_exported_total{node_id} +airgap_bundles_imported_total{node_id} +airgap_jobs_synced_total{node_id} +airgap_duplicates_dropped_total{node_id} +airgap_merge_conflicts_total{conflict_type} + +# Histograms +airgap_bundle_size_bytes{node_id} +airgap_sync_duration_seconds{node_id} +airgap_merge_entries_count{node_id} + +# Gauges +airgap_pending_sync_bundles{node_id} +airgap_last_sync_timestamp{node_id} +``` + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Merge sorts by full HLC tuple | Ensures total order even with identical physical time | +| Recompute chain on merge | Central chain must be contiguous; original links preserved for audit | +| Store source node ID | Traceability for sync origin | +| Error on payload mismatch | Same JobId must have same payload (determinism invariant) | + +| Risk | Mitigation | +|------|------------| +| Large bundle sizes | Compression; chunked sync; incremental bundles | +| Clock skew exceeds HLC tolerance | Pre-sync clock validation; NTP enforcement | +| Merge performance with many nodes | Parallel sort; streaming merge; batch processing | +| Bundle corruption during transfer | DSSE signature; checksum validation | + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +## Next Checkpoints + +- 2026-01-12: OMP-001 to OMP-007 complete (core merge) +- 2026-01-13: OMP-008 to OMP-013 complete (bundle + CLI) +- 2026-01-14: OMP-014 to OMP-021 complete (tests, docs) diff --git a/docs/implplan/SPRINT_20260105_002_004_BE_hlc_integration_tests.md b/docs/implplan/SPRINT_20260105_002_004_BE_hlc_integration_tests.md new file mode 100644 index 000000000..ba85325a4 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_004_BE_hlc_integration_tests.md @@ -0,0 +1,515 @@ +# Sprint 20260105_002_004_BE - HLC: Cross-Module Integration & Testing + +## Topic & Scope + +Comprehensive integration testing and cross-module wiring for the HLC-based audit-safe job queue ordering implementation. Ensures end-to-end determinism from job enqueue through verdict replay. + +- **Working directory:** `src/__Tests/Integration/`, cross-module +- **Evidence:** Integration test suite, E2E tests, performance benchmarks, documentation + +## Problem Statement + +Individual HLC components (library, scheduler chain, offline merge) must work together seamlessly: +- HLC timestamps must flow through Scheduler -> Timeline -> Ledgers +- Chain links must be verifiable across module boundaries +- Batch snapshots must integrate with Attestor for DSSE signing +- Replay must produce identical results with HLC-ordered inputs + +## Dependencies & Concurrency + +- **Depends on:** All previous sprints (002_001, 002_002, 002_003) +- **Blocks:** Production rollout +- **Parallel safe:** Test development can proceed once interfaces are defined + +## Documentation Prerequisites + +- All previous sprint documentation +- docs/modules/attestor/proof-chain-specification.md +- docs/modules/replay/architecture.md +- src/__Tests/AGENTS.md + +## Technical Design + +### Cross-Module HLC Flow + +``` +┌─────────────┐ HLC Tick ┌─────────────┐ Chain Link ┌─────────────┐ +│ Client │ ───────────────▶│ Scheduler │ ─────────────────▶│ Scheduler │ +│ Request │ │ Queue │ │ Log │ +└─────────────┘ └─────────────┘ └─────────────┘ + │ │ + │ Job Execution │ + ▼ │ + ┌─────────────┐ │ + │ Orchestrator│ │ + │ Job │ │ + └─────────────┘ │ + │ │ + │ Evidence │ + ▼ │ +┌─────────────┐ Batch Snap ┌─────────────┐ DSSE Sign ┌─────────────┐ +│ Replay │ ◀──────────────│ Findings │ ◀────────────────│ Attestor │ +│ Engine │ │ Ledger │ │ │ +└─────────────┘ └─────────────┘ └─────────────┘ +``` + +### Integration Test Categories + +#### 1. HLC Propagation Tests + +```csharp +[Trait("Category", "Integration")] +[Trait("Category", "HLC")] +public sealed class HlcPropagationTests : IClassFixture +{ + [Fact] + public async Task Enqueue_HlcTimestamp_PropagatedToTimeline() + { + // Arrange + var job = CreateTestJob(); + + // Act + var enqueueResult = await _scheduler.EnqueueAsync(job); + await WaitForTimelineEventAsync(enqueueResult.JobId); + + // Assert + var timelineEvent = await _timeline.GetByCorrelationIdAsync(enqueueResult.JobId); + Assert.NotNull(timelineEvent); + Assert.Equal(enqueueResult.THlc.ToSortableString(), timelineEvent.HlcTimestamp); + } + + [Fact] + public async Task Enqueue_HlcTimestamp_PropagatedToFindingsLedger() + { + // Arrange + var job = CreateScanJob(); + + // Act + var enqueueResult = await _scheduler.EnqueueAsync(job); + await WaitForJobCompletionAsync(enqueueResult.JobId); + + // Assert + var ledgerEvent = await _findingsLedger.GetBySourceRunIdAsync(enqueueResult.JobId); + Assert.NotNull(ledgerEvent); + Assert.True(HlcTimestamp.Parse(ledgerEvent.HlcTimestamp) >= enqueueResult.THlc); + } +} +``` + +#### 2. Chain Integrity Tests + +```csharp +[Trait("Category", "Integration")] +[Trait("Category", "HLC")] +public sealed class ChainIntegrityTests : IClassFixture +{ + [Fact] + public async Task EnqueueMultiple_ChainLinksValid() + { + // Arrange + var jobs = Enumerable.Range(0, 100).Select(i => CreateTestJob(i)).ToList(); + + // Act + foreach (var job in jobs) + { + await _scheduler.EnqueueAsync(job); + } + + // Assert + var verificationResult = await _scheduler.VerifyChainIntegrityAsync(_tenantId); + Assert.True(verificationResult.IsValid); + Assert.Equal(100, verificationResult.EntriesChecked); + Assert.Empty(verificationResult.Issues); + } + + [Fact] + public async Task ChainVerification_DetectsTampering() + { + // Arrange + var jobs = Enumerable.Range(0, 10).Select(i => CreateTestJob(i)).ToList(); + foreach (var job in jobs) + { + await _scheduler.EnqueueAsync(job); + } + + // Act - Tamper with middle entry + await TamperWithSchedulerLogEntryAsync(jobs[5].Id); + + // Assert + var verificationResult = await _scheduler.VerifyChainIntegrityAsync(_tenantId); + Assert.False(verificationResult.IsValid); + Assert.Contains(verificationResult.Issues, i => i.JobId == jobs[5].Id); + } +} +``` + +#### 3. Batch Snapshot Integration + +```csharp +[Trait("Category", "Integration")] +[Trait("Category", "HLC")] +public sealed class BatchSnapshotIntegrationTests : IClassFixture +{ + [Fact] + public async Task CreateSnapshot_SignedByAttestor() + { + // Arrange + var jobs = await EnqueueMultipleJobsAsync(50); + var startT = jobs.First().THlc; + var endT = jobs.Last().THlc; + + // Act + var snapshot = await _batchService.CreateSnapshotAsync(_tenantId, startT, endT); + + // Assert + Assert.NotNull(snapshot.Signature); + Assert.NotNull(snapshot.SignedBy); + + var verified = await _attestor.VerifySnapshotSignatureAsync(snapshot); + Assert.True(verified); + } + + [Fact] + public async Task Snapshot_HeadLinkMatchesChain() + { + // Arrange + var jobs = await EnqueueMultipleJobsAsync(25); + + // Act + var snapshot = await _batchService.CreateSnapshotAsync( + _tenantId, + jobs.First().THlc, + jobs.Last().THlc); + + // Assert + var chainHead = await _schedulerLog.GetChainHeadAsync(_tenantId); + Assert.Equal(chainHead.Link, snapshot.HeadLink); + } +} +``` + +#### 4. Offline Sync Integration + +```csharp +[Trait("Category", "Integration")] +[Trait("Category", "HLC")] +[Trait("Category", "AirGap")] +public sealed class OfflineSyncIntegrationTests : IClassFixture +{ + [Fact] + public async Task OfflineEnqueue_SyncsWithCorrectOrder() + { + // Arrange - Simulate two offline nodes + var nodeA = CreateOfflineNode("node-a"); + var nodeB = CreateOfflineNode("node-b"); + + // Enqueue interleaved jobs + await nodeA.EnqueueAsync(CreateJob("A1")); // T=100 + await nodeB.EnqueueAsync(CreateJob("B1")); // T=101 + await nodeA.EnqueueAsync(CreateJob("A2")); // T=102 + + // Act - Export and sync + var bundleA = await nodeA.ExportBundleAsync(); + var bundleB = await nodeB.ExportBundleAsync(); + var syncResult = await _syncService.SyncFromBundlesAsync([bundleA, bundleB]); + + // Assert - Merged in HLC order + var merged = await _schedulerLog.GetByHlcOrderAsync(_tenantId, limit: 10); + Assert.Equal(3, merged.Count); + Assert.Equal("A1", merged[0].JobName); // T=100 + Assert.Equal("B1", merged[1].JobName); // T=101 + Assert.Equal("A2", merged[2].JobName); // T=102 + } + + [Fact] + public async Task OfflineSync_DeduplicatesSamePayload() + { + // Arrange - Same job enqueued on two nodes + var nodeA = CreateOfflineNode("node-a"); + var nodeB = CreateOfflineNode("node-b"); + + var samePayload = CreateJob("shared"); + await nodeA.EnqueueAsync(samePayload); + await nodeB.EnqueueAsync(samePayload); // Same payload = same JobId + + // Act + var bundleA = await nodeA.ExportBundleAsync(); + var bundleB = await nodeB.ExportBundleAsync(); + var syncResult = await _syncService.SyncFromBundlesAsync([bundleA, bundleB]); + + // Assert + Assert.Equal(1, syncResult.Appended); + Assert.Equal(1, syncResult.Duplicates); + } +} +``` + +#### 5. Replay Determinism Tests + +```csharp +[Trait("Category", "Integration")] +[Trait("Category", "HLC")] +[Trait("Category", "Replay")] +public sealed class HlcReplayDeterminismTests : IClassFixture +{ + [Fact] + public async Task Replay_SameHlcOrder_SameResults() + { + // Arrange + var jobs = await EnqueueAndExecuteJobsAsync(20); + var snapshot = await _batchService.CreateSnapshotAsync( + _tenantId, + jobs.First().THlc, + jobs.Last().THlc); + + var originalResults = await GetJobResultsAsync(jobs); + + // Act - Replay with same HLC-ordered inputs + var replayResults = await _replayEngine.ReplayFromSnapshotAsync(snapshot); + + // Assert + Assert.Equal(originalResults.Count, replayResults.Count); + for (int i = 0; i < originalResults.Count; i++) + { + Assert.Equal(originalResults[i].VerdictDigest, replayResults[i].VerdictDigest); + } + } + + [Fact] + public async Task Replay_HlcOrderPreserved_AcrossRestarts() + { + // Arrange + var jobs = await EnqueueJobsAsync(10); + var hlcOrder = jobs.Select(j => j.THlc.ToSortableString()).ToList(); + + // Act - Simulate restart + await RestartSchedulerServiceAsync(); + var recoveredJobs = await _schedulerLog.GetByHlcOrderAsync(_tenantId, limit: 10); + + // Assert + var recoveredOrder = recoveredJobs.Select(j => j.THlc).ToList(); + Assert.Equal(hlcOrder, recoveredOrder); + } +} +``` + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owner | Task Definition | +|---|---------|--------|------------|-------|-----------------| +| 1 | INT-001 | TODO | All sprints | Guild | Create `StellaOps.Integration.HLC` test project | +| 2 | INT-002 | TODO | INT-001 | Guild | Implement `HlcTestFixture` with full stack setup | +| 3 | INT-003 | TODO | INT-002 | Guild | Write HLC propagation tests (Scheduler -> Timeline) | +| 4 | INT-004 | TODO | INT-003 | Guild | Write HLC propagation tests (Scheduler -> Ledger) | +| 5 | INT-005 | TODO | INT-004 | Guild | Write chain integrity tests (valid chain) | +| 6 | INT-006 | TODO | INT-005 | Guild | Write chain integrity tests (tampering detection) | +| 7 | INT-007 | TODO | INT-006 | Guild | Write batch snapshot + Attestor integration tests | +| 8 | INT-008 | TODO | INT-007 | Guild | Create `AirGapTestFixture` for offline simulation | +| 9 | INT-009 | TODO | INT-008 | Guild | Write offline sync integration tests (order) | +| 10 | INT-010 | TODO | INT-009 | Guild | Write offline sync integration tests (dedup) | +| 11 | INT-011 | TODO | INT-010 | Guild | Write offline sync integration tests (multi-node) | +| 12 | INT-012 | TODO | INT-011 | Guild | Write replay determinism tests | +| 13 | INT-013 | TODO | INT-012 | Guild | Write E2E test: full job lifecycle with HLC | +| 14 | INT-014 | TODO | INT-013 | Guild | Write performance benchmarks: HLC tick throughput | +| 15 | INT-015 | TODO | INT-014 | Guild | Write performance benchmarks: chain verification | +| 16 | INT-016 | TODO | INT-015 | Guild | Write performance benchmarks: offline merge | +| 17 | INT-017 | TODO | INT-016 | Guild | Create Grafana dashboard for HLC metrics | +| 18 | INT-018 | TODO | INT-017 | Guild | Create alerts for HLC anomalies | +| 19 | INT-019 | TODO | INT-018 | Guild | Update Architecture documentation | +| 20 | INT-020 | TODO | INT-019 | Guild | Create Operations runbook for HLC | +| 21 | INT-021 | TODO | INT-020 | Guild | Create Migration guide for existing deployments | +| 22 | INT-022 | TODO | INT-021 | Guild | Final review and sign-off | + +## Performance Benchmarks + +### Benchmark Targets + +| Metric | Target | Rationale | +|--------|--------|-----------| +| HLC tick throughput | > 100K/sec | Support high-volume job queues | +| Chain link computation | < 10us | Minimal overhead per enqueue | +| Chain verification (1K entries) | < 100ms | Fast audit checks | +| Offline merge (10K entries) | < 1s | Reasonable sync time | +| Batch snapshot creation | < 500ms | Interactive batch operations | + +### Benchmark Suite + +```csharp +[MemoryDiagnoser] +[SimpleJob(RuntimeMoniker.Net100)] +public class HlcBenchmarks +{ + private IHybridLogicalClock _hlc = null!; + + [GlobalSetup] + public void Setup() + { + _hlc = new HybridLogicalClock( + TimeProvider.System, + "bench-node", + new InMemoryHlcStateStore()); + } + + [Benchmark] + public HlcTimestamp Tick() => _hlc.Tick(); + + [Benchmark] + public byte[] ComputeChainLink() + { + return SchedulerChainLinking.ComputeLink( + _prevLink, + _jobId, + _hlc.Tick(), + _payloadHash); + } + + [Benchmark] + [Arguments(100)] + [Arguments(1000)] + [Arguments(10000)] + public async Task VerifyChain(int entries) + { + await _verifier.VerifyAsync(_tenantId, entries); + } +} +``` + +## Observability Integration + +### Grafana Dashboard Panels + +```json +{ + "panels": [ + { + "title": "HLC Ticks per Second", + "expr": "rate(hlc_ticks_total[1m])" + }, + { + "title": "HLC Clock Skew Rejections", + "expr": "rate(hlc_clock_skew_rejections_total[5m])" + }, + { + "title": "Scheduler Chain Verifications", + "expr": "rate(scheduler_chain_verifications_total[5m])" + }, + { + "title": "Chain Verification Failures", + "expr": "scheduler_chain_verification_failures_total" + }, + { + "title": "AirGap Sync Duration P99", + "expr": "histogram_quantile(0.99, airgap_sync_duration_seconds_bucket)" + }, + { + "title": "Batch Snapshots Created", + "expr": "rate(scheduler_batch_snapshots_total[1h])" + } + ] +} +``` + +### Alerts + +```yaml +groups: + - name: hlc-alerts + rules: + - alert: HlcClockSkewExcessive + expr: rate(hlc_clock_skew_rejections_total[5m]) > 0 + for: 1m + labels: + severity: warning + annotations: + summary: "HLC clock skew rejections detected" + description: "Node {{ $labels.node_id }} is rejecting timestamps due to clock skew" + + - alert: SchedulerChainCorruption + expr: scheduler_chain_verification_failures_total > 0 + for: 0m + labels: + severity: critical + annotations: + summary: "Scheduler chain corruption detected" + description: "Chain verification failed for tenant {{ $labels.tenant_id }}" + + - alert: AirGapSyncBacklog + expr: airgap_pending_sync_bundles > 10 + for: 10m + labels: + severity: warning + annotations: + summary: "AirGap sync backlog growing" +``` + +## Documentation Updates + +### Files to Update + +| File | Changes | +|------|---------| +| `docs/ARCHITECTURE_REFERENCE.md` | Add HLC section | +| `docs/modules/scheduler/architecture.md` | Document HLC ordering | +| `docs/airgap/OFFLINE_KIT.md` | Add HLC merge protocol | +| `docs/observability/observability.md` | Add HLC metrics | +| `docs/operations/runbooks/` | Create `hlc-troubleshooting.md` | +| `CLAUDE.md` | Add HLC guidelines to Section 8 | + +### CLAUDE.md Update (Section 8.19) + +```markdown +### 8.19) HLC Usage for Audit-Safe Ordering + +| Rule | Guidance | +|------|----------| +| **Use HLC for distributed ordering** | When ordering must be deterministic across distributed nodes or offline scenarios, use `IHybridLogicalClock.Tick()` instead of `TimeProvider.GetUtcNow()`. | + +```csharp +// BAD - wall-clock ordering, susceptible to skew +var timestamp = _timeProvider.GetUtcNow(); +var job = new Job { CreatedAt = timestamp }; + +// GOOD - HLC ordering, skew-resistant +var hlcTimestamp = _hlc.Tick(); +var job = new Job { THlc = hlcTimestamp }; +``` +``` + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Separate integration test project | Isolation from unit tests; different dependencies | +| Testcontainers for Postgres | Realistic integration without mocking | +| Benchmark suite in main repo | Track performance over time; CI integration | +| Grafana dashboard as code | Version-controlled observability | + +| Risk | Mitigation | +|------|------------| +| Integration test flakiness | Retry logic; deterministic test data; container health checks | +| Performance regression | Benchmark baselines; CI gates on performance | +| Documentation drift | Doc updates required for PR merge | + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +## Next Checkpoints + +- 2026-01-15: INT-001 to INT-007 complete (basic integration) +- 2026-01-16: INT-008 to INT-013 complete (offline + replay) +- 2026-01-17: INT-014 to INT-022 complete (perf, docs, rollout) + +## Final Acceptance Criteria + +- [ ] All integration tests pass in CI +- [ ] Chain verification detects 100% of tampering attempts +- [ ] Offline merge produces deterministic results +- [ ] Replay with HLC inputs produces identical outputs +- [ ] Performance benchmarks meet targets +- [ ] Grafana dashboard deployed +- [ ] Alerts configured and tested +- [ ] Documentation complete +- [ ] Migration guide validated on staging diff --git a/docs/implplan/SPRINT_20260105_002_004_CLI_seal_drift_commands.md b/docs/implplan/SPRINT_20260105_002_004_CLI_seal_drift_commands.md new file mode 100644 index 000000000..e5ba85a69 --- /dev/null +++ b/docs/implplan/SPRINT_20260105_002_004_CLI_seal_drift_commands.md @@ -0,0 +1,1013 @@ +# Sprint 20260105_002_004_CLI - Facet CLI Commands & Admission Integration + +## Topic & Scope + +Implement the CLI commands for facet operations (`stella seal`, `stella drift`, `stella vex gen --from-drift`) and integrate facet seal verification into the Zastava admission webhook. This sprint delivers the user-facing tooling for the facet sealing feature. + +**Advisory Reference:** Product advisory - CLI/API sections for `stella seal`, `stella drift`, `stella vex gen --from-drift`. + +**Key Insight:** Users need simple CLI commands to seal images at admission, check drift on deploy, and generate VEX statements for authorized changes. The admission webhook provides automated enforcement. + +**Working directory:** `src/Cli/`, `src/Zastava/` + +**Evidence:** Functional `stella seal`, `stella drift`, `stella vex gen --from-drift` commands, admission webhook with facet verification. + +--- + +## Dependencies & Concurrency + +| Dependency | Type | Status | +|------------|------|--------| +| SPRINT_20260105_002_002_FACET | Sprint | Required | +| SPRINT_20260105_002_003_FACET | Sprint | Required | +| Zastava Admission Webhook | Internal | Available | +| CLI Infrastructure | Internal | Available | +| IFacetExtractor | Sprint 002 | Required | +| IFacetDriftEngine | Sprint 003 | Required | + +**Parallel Execution:** CLI commands (CLI-001 through CLI-015) and admission integration (ADM-001 through ADM-010) can proceed in parallel once dependencies from Sprints 002/003 are met. + +--- + +## Documentation Prerequisites + +- SPRINT_20260105_002_002_FACET models +- SPRINT_20260105_002_003_FACET drift engine +- `src/Cli/StellaOps.Cli/Commands/` existing patterns +- `src/Zastava/__Tests/StellaOps.Zastava.Webhook.Tests/Admission/` +- `docs/cli/admin-reference.md` + +--- + +## Problem Analysis + +### Current State + +StellaOps currently: +- Has CLI infrastructure with command groups +- Has Zastava admission webhook with policy checking +- Has surface manifest validation at admission +- No `stella seal` command +- No `stella drift` command +- No `stella vex gen --from-drift` command +- No facet verification in admission + +**Gaps:** +1. Missing CLI commands for facet operations +2. Admission doesn't verify facet seals +3. No VEX generation workflow from drift + +### Target Capabilities + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CLI Commands for Facet Operations │ +│ │ +│ stella seal --image sha256:abc... │ +│ ├── Pull image layers │ +│ ├── Extract facets via IFacetExtractor │ +│ ├── Compute per-facet Merkle roots │ +│ ├── Sign FacetSeal with DSSE │ +│ ├── Store seal (local file or remote API) │ +│ └── Output: seal ID and summary │ +│ │ +│ stella drift --image sha256:abc... │ +│ ├── Load baseline seal │ +│ ├── Pull current image │ +│ ├── Compute drift via IFacetDriftEngine │ +│ ├── Evaluate quotas │ +│ └── Output: per-facet drift report with verdicts │ +│ │ +│ stella vex gen --from-drift --image sha256:abc... │ +│ ├── Compute drift (same as stella drift) │ +│ ├── Generate VEX drafts for facets requiring authorization │ +│ ├── Write to stdout or file │ +│ └── Output: OpenVEX document(s) │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────┐ │ +│ │ Admission Integration │ │ +│ │ │ │ +│ │ Zastava Webhook receives admission request │ │ +│ │ ├── Check if facet sealing is required (namespace annotation) │ │ +│ │ ├── Load FacetSeal for image │ │ +│ │ ├── Verify seal signature │ │ +│ │ ├── Compute drift against current image │ │ +│ │ ├── Evaluate facet quotas │ │ +│ │ └── Admit/Reject based on verdict │ │ +│ │ │ │ +│ └─────────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## Architecture Design + +### stella seal Command + +```csharp +// src/Cli/StellaOps.Cli/Commands/SealCommandGroup.cs +namespace StellaOps.Cli.Commands; + +/// +/// Command group for facet sealing operations. +/// +internal static class SealCommandGroup +{ + public static Command CreateSealCommand() + { + var imageOption = new Option( + "--image", + "Image reference or digest to seal") + { IsRequired = true }; + + var outputOption = new Option( + "--output", + "Output file path for seal (default: stdout)"); + + var storeOption = new Option( + "--store", + () => true, + "Store seal in remote API"); + + var signOption = new Option( + "--sign", + () => true, + "Sign seal with DSSE"); + + var keyOption = new Option( + "--key", + "Private key path for signing (default: use configured key)"); + + var facetsOption = new Option( + "--facets", + "Specific facets to seal (default: all)"); + + var formatOption = new Option( + "--format", + () => "json", + "Output format: json, yaml, compact"); + + var command = new Command("seal", "Create facet seal for an image") + { + imageOption, + outputOption, + storeOption, + signOption, + keyOption, + facetsOption, + formatOption + }; + + command.SetHandler(async (context) => + { + var image = context.ParseResult.GetValueForOption(imageOption)!; + var output = context.ParseResult.GetValueForOption(outputOption); + var store = context.ParseResult.GetValueForOption(storeOption); + var sign = context.ParseResult.GetValueForOption(signOption); + var key = context.ParseResult.GetValueForOption(keyOption); + var facets = context.ParseResult.GetValueForOption(facetsOption); + var format = context.ParseResult.GetValueForOption(formatOption)!; + var ct = context.GetCancellationToken(); + + await HandleSealAsync( + context.BindingContext.GetRequiredService(), + image, output, store, sign, key, facets, format, ct) + .ConfigureAwait(false); + }); + + return command; + } + + private static async Task HandleSealAsync( + IServiceProvider services, + string image, + string? outputPath, + bool store, + bool sign, + string? keyPath, + string[]? facets, + string format, + CancellationToken ct) + { + using var activity = CliActivitySource.Instance.StartActivity("cli.seal"); + + var imageResolver = services.GetRequiredService(); + var facetExtractor = services.GetRequiredService(); + var merkleTree = services.GetRequiredService(); + var sealStore = services.GetService(); + var signer = services.GetService(); + var timeProvider = services.GetRequiredService(); + + AnsiConsole.Status() + .Start("Resolving image...", async ctx => + { + // 1. Resolve image + ctx.Status("Resolving image..."); + var imageInfo = await imageResolver.ResolveAsync(image, ct).ConfigureAwait(false); + AnsiConsole.MarkupLine($"[green]Resolved:[/] {imageInfo.Digest}"); + + // 2. Determine facets to seal + var facetsToSeal = facets?.Length > 0 + ? BuiltInFacets.All.Where(f => facets.Contains(f.FacetId)).ToList() + : BuiltInFacets.All.ToList(); + + ctx.Status($"Extracting {facetsToSeal.Count} facets..."); + var facetEntries = new List(); + + foreach (var facet in facetsToSeal) + { + ct.ThrowIfCancellationRequested(); + ctx.Status($"Extracting facet: {facet.Name}..."); + + var extraction = await facetExtractor.ExtractAsync( + facet, + imageInfo.FileSystem, + new FacetExtractionOptions { ComputeHashes = true }, + ct).ConfigureAwait(false); + + if (extraction.Files.Length == 0) + { + AnsiConsole.MarkupLine($"[dim]Skipping empty facet: {facet.Name}[/]"); + continue; + } + + var merkleRoot = merkleTree.ComputeRoot(extraction.Files); + + facetEntries.Add(new FacetEntry + { + FacetId = facet.FacetId, + Name = facet.Name, + Category = facet.Category, + Selectors = [.. facet.Selectors], + MerkleRoot = merkleRoot, + FileCount = extraction.Files.Length, + TotalBytes = extraction.TotalBytes, + Files = extraction.Files + }); + + AnsiConsole.MarkupLine($"[green]Sealed:[/] {facet.Name} ({extraction.Files.Length} files, {FormatBytes(extraction.TotalBytes)})"); + } + + // 3. Create seal + ctx.Status("Creating seal..."); + var combinedRoot = merkleTree.ComputeCombinedRoot(facetEntries); + + var seal = new FacetSeal + { + ImageDigest = imageInfo.Digest, + CreatedAt = timeProvider.GetUtcNow(), + Facets = [.. facetEntries], + CombinedMerkleRoot = combinedRoot + }; + + // 4. Sign if requested + if (sign && signer is not null) + { + ctx.Status("Signing seal..."); + var canonical = CanonicalJsonSerializer.Serialize(seal); + var signature = await signer.SignAsync( + "application/vnd.stellaops.facetseal+json", + Encoding.UTF8.GetBytes(canonical), + keyPath, + ct).ConfigureAwait(false); + seal = seal with { Signature = signature }; + AnsiConsole.MarkupLine("[green]Signed[/]"); + } + + // 5. Store if requested + if (store && sealStore is not null) + { + ctx.Status("Storing seal..."); + await sealStore.SaveAsync(seal, ct).ConfigureAwait(false); + AnsiConsole.MarkupLine("[green]Stored to API[/]"); + } + + // 6. Output + var output = FormatSeal(seal, format); + if (!string.IsNullOrEmpty(outputPath)) + { + await File.WriteAllTextAsync(outputPath, output, ct).ConfigureAwait(false); + AnsiConsole.MarkupLine($"[green]Written to:[/] {outputPath}"); + } + else + { + Console.WriteLine(output); + } + + // Summary + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine($"[bold]Seal Summary[/]"); + AnsiConsole.MarkupLine($" Image: {imageInfo.Digest}"); + AnsiConsole.MarkupLine($" Facets: {facetEntries.Count}"); + AnsiConsole.MarkupLine($" Combined Root: {combinedRoot}"); + }); + } + + private static string FormatSeal(FacetSeal seal, string format) + { + return format.ToLowerInvariant() switch + { + "json" => JsonSerializer.Serialize(seal, new JsonSerializerOptions { WriteIndented = true }), + "yaml" => ToYaml(seal), + "compact" => $"{seal.ImageDigest}|{seal.CombinedMerkleRoot}|{seal.Facets.Length}", + _ => JsonSerializer.Serialize(seal, new JsonSerializerOptions { WriteIndented = true }) + }; + } + + private static string ToYaml(FacetSeal seal) + { + // Simplified YAML output + var sb = new StringBuilder(); + sb.AppendLine($"imageDigest: {seal.ImageDigest}"); + sb.AppendLine($"createdAt: {seal.CreatedAt:O}"); + sb.AppendLine($"combinedMerkleRoot: {seal.CombinedMerkleRoot}"); + sb.AppendLine("facets:"); + foreach (var f in seal.Facets) + { + sb.AppendLine($" - facetId: {f.FacetId}"); + sb.AppendLine($" merkleRoot: {f.MerkleRoot}"); + sb.AppendLine($" fileCount: {f.FileCount}"); + } + return sb.ToString(); + } + + private static string FormatBytes(long bytes) + { + return bytes switch + { + < 1024 => $"{bytes} B", + < 1024 * 1024 => $"{bytes / 1024.0:F1} KB", + < 1024 * 1024 * 1024 => $"{bytes / (1024.0 * 1024):F1} MB", + _ => $"{bytes / (1024.0 * 1024 * 1024):F1} GB" + }; + } +} +``` + +### stella drift Command + +```csharp +// src/Cli/StellaOps.Cli/Commands/DriftCommandGroup.cs +namespace StellaOps.Cli.Commands; + +/// +/// Command group for drift analysis operations. +/// +internal static class DriftCommandGroup +{ + public static Command CreateDriftCommand() + { + var imageOption = new Option( + "--image", + "Image reference or digest to analyze") + { IsRequired = true }; + + var baselineOption = new Option( + "--baseline", + "Baseline seal ID (default: latest for image)"); + + var formatOption = new Option( + "--format", + () => "table", + "Output format: table, json, yaml"); + + var verboseOption = new Option( + "--verbose", + () => false, + "Show detailed file changes"); + + var failOnBreachOption = new Option( + "--fail-on-breach", + () => false, + "Exit with error code if quota breached"); + + var command = new Command("drift", "Analyze facet drift against baseline seal") + { + imageOption, + baselineOption, + formatOption, + verboseOption, + failOnBreachOption + }; + + command.SetHandler(async (context) => + { + var image = context.ParseResult.GetValueForOption(imageOption)!; + var baseline = context.ParseResult.GetValueForOption(baselineOption); + var format = context.ParseResult.GetValueForOption(formatOption)!; + var verbose = context.ParseResult.GetValueForOption(verboseOption); + var failOnBreach = context.ParseResult.GetValueForOption(failOnBreachOption); + var ct = context.GetCancellationToken(); + + await HandleDriftAsync( + context.BindingContext.GetRequiredService(), + image, baseline, format, verbose, failOnBreach, ct) + .ConfigureAwait(false); + }); + + return command; + } + + private static async Task HandleDriftAsync( + IServiceProvider services, + string image, + string? baselineId, + string format, + bool verbose, + bool failOnBreach, + CancellationToken ct) + { + using var activity = CliActivitySource.Instance.StartActivity("cli.drift"); + + var imageResolver = services.GetRequiredService(); + var driftEngine = services.GetRequiredService(); + var sealStore = services.GetRequiredService(); + + // 1. Resolve image + AnsiConsole.Status() + .Start("Analyzing drift...", async ctx => + { + ctx.Status("Resolving image..."); + var imageInfo = await imageResolver.ResolveAsync(image, ct).ConfigureAwait(false); + + // 2. Load baseline + ctx.Status("Loading baseline seal..."); + FacetSeal? baseline; + if (!string.IsNullOrEmpty(baselineId)) + { + baseline = await sealStore.GetBySealIdAsync(baselineId, ct).ConfigureAwait(false); + } + else + { + baseline = await sealStore.GetLatestSealAsync(imageInfo.Digest, ct).ConfigureAwait(false); + } + + if (baseline is null) + { + AnsiConsole.MarkupLine("[red]No baseline seal found[/]"); + Environment.ExitCode = 1; + return; + } + + AnsiConsole.MarkupLine($"[green]Baseline:[/] {baseline.CombinedMerkleRoot}"); + + // 3. Compute drift + ctx.Status("Computing drift..."); + var report = await driftEngine.ComputeDriftAsync( + baseline, + imageInfo.FileSystem, + ct: ct).ConfigureAwait(false); + + // 4. Output + OutputDriftReport(report, format, verbose); + + // 5. Exit code + if (failOnBreach && report.OverallVerdict == QuotaVerdict.Blocked) + { + Environment.ExitCode = 2; + } + }); + } + + private static void OutputDriftReport(FacetDriftReport report, string format, bool verbose) + { + if (format == "json") + { + Console.WriteLine(JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true })); + return; + } + + if (format == "yaml") + { + OutputYaml(report); + return; + } + + // Table format + AnsiConsole.WriteLine(); + var verdictColor = report.OverallVerdict switch + { + QuotaVerdict.Ok => "green", + QuotaVerdict.Warning => "yellow", + QuotaVerdict.Blocked => "red", + QuotaVerdict.RequiresVex => "blue", + _ => "white" + }; + AnsiConsole.MarkupLine($"[bold]Overall Verdict:[/] [{verdictColor}]{report.OverallVerdict}[/]"); + AnsiConsole.MarkupLine($"[bold]Total Changed Files:[/] {report.TotalChangedFiles}"); + AnsiConsole.WriteLine(); + + var table = new Table() + .AddColumn("Facet") + .AddColumn("Added") + .AddColumn("Removed") + .AddColumn("Modified") + .AddColumn("Churn %") + .AddColumn("Verdict"); + + foreach (var drift in report.FacetDrifts) + { + var vColor = drift.QuotaVerdict switch + { + QuotaVerdict.Ok => "green", + QuotaVerdict.Warning => "yellow", + QuotaVerdict.Blocked => "red", + QuotaVerdict.RequiresVex => "blue", + _ => "white" + }; + + table.AddRow( + drift.FacetId, + drift.Added.Length.ToString(), + drift.Removed.Length.ToString(), + drift.Modified.Length.ToString(), + $"{drift.ChurnPercent:F1}%", + $"[{vColor}]{drift.QuotaVerdict}[/]"); + } + + AnsiConsole.Write(table); + + if (verbose) + { + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]File Changes:[/]"); + foreach (var drift in report.FacetDrifts.Where(d => d.Added.Length + d.Removed.Length + d.Modified.Length > 0)) + { + AnsiConsole.MarkupLine($"\n[bold]{drift.FacetId}[/]"); + foreach (var f in drift.Added.Take(10)) + AnsiConsole.MarkupLine($" [green]+[/] {f.Path}"); + foreach (var f in drift.Removed.Take(10)) + AnsiConsole.MarkupLine($" [red]-[/] {f.Path}"); + foreach (var f in drift.Modified.Take(10)) + AnsiConsole.MarkupLine($" [yellow]~[/] {f.Path}"); + + var total = drift.Added.Length + drift.Removed.Length + drift.Modified.Length; + if (total > 30) + AnsiConsole.MarkupLine($" [dim]... and {total - 30} more[/]"); + } + } + } + + private static void OutputYaml(FacetDriftReport report) + { + var sb = new StringBuilder(); + sb.AppendLine($"imageDigest: {report.ImageDigest}"); + sb.AppendLine($"overallVerdict: {report.OverallVerdict}"); + sb.AppendLine($"totalChangedFiles: {report.TotalChangedFiles}"); + sb.AppendLine("facetDrifts:"); + foreach (var d in report.FacetDrifts) + { + sb.AppendLine($" - facetId: {d.FacetId}"); + sb.AppendLine($" added: {d.Added.Length}"); + sb.AppendLine($" removed: {d.Removed.Length}"); + sb.AppendLine($" modified: {d.Modified.Length}"); + sb.AppendLine($" churnPercent: {d.ChurnPercent:F2}"); + sb.AppendLine($" verdict: {d.QuotaVerdict}"); + } + Console.WriteLine(sb.ToString()); + } +} +``` + +### stella vex gen --from-drift Command + +```csharp +// src/Cli/StellaOps.Cli/Commands/VexGenCommandGroup.cs +namespace StellaOps.Cli.Commands; + +/// +/// Command group for VEX generation operations. +/// +internal static class VexGenCommandGroup +{ + public static Command CreateVexGenCommand() + { + var fromDriftOption = new Option( + "--from-drift", + "Generate VEX from facet drift analysis") + { IsRequired = true }; + + var imageOption = new Option( + "--image", + "Image reference or digest") + { IsRequired = true }; + + var baselineOption = new Option( + "--baseline", + "Baseline seal ID (default: latest)"); + + var outputOption = new Option( + "--output", + "Output file path (default: stdout)"); + + var formatOption = new Option( + "--format", + () => "openvex", + "VEX format: openvex, csaf"); + + var statusOption = new Option( + "--status", + () => "under_investigation", + "VEX status: under_investigation, not_affected, affected"); + + var command = new Command("gen", "Generate VEX statements") + { + fromDriftOption, + imageOption, + baselineOption, + outputOption, + formatOption, + statusOption + }; + + command.SetHandler(async (context) => + { + var fromDrift = context.ParseResult.GetValueForOption(fromDriftOption); + var image = context.ParseResult.GetValueForOption(imageOption)!; + var baseline = context.ParseResult.GetValueForOption(baselineOption); + var output = context.ParseResult.GetValueForOption(outputOption); + var format = context.ParseResult.GetValueForOption(formatOption)!; + var status = context.ParseResult.GetValueForOption(statusOption)!; + var ct = context.GetCancellationToken(); + + if (fromDrift) + { + await HandleVexFromDriftAsync( + context.BindingContext.GetRequiredService(), + image, baseline, output, format, status, ct) + .ConfigureAwait(false); + } + }); + + return command; + } + + private static async Task HandleVexFromDriftAsync( + IServiceProvider services, + string image, + string? baselineId, + string? outputPath, + string format, + string status, + CancellationToken ct) + { + using var activity = CliActivitySource.Instance.StartActivity("cli.vex.gen.drift"); + + var imageResolver = services.GetRequiredService(); + var driftEngine = services.GetRequiredService(); + var sealStore = services.GetRequiredService(); + var vexEmitter = services.GetRequiredService(); + var timeProvider = services.GetRequiredService(); + var guidProvider = services.GetRequiredService(); + + AnsiConsole.Status() + .Start("Generating VEX from drift...", async ctx => + { + // 1. Resolve image + ctx.Status("Resolving image..."); + var imageInfo = await imageResolver.ResolveAsync(image, ct).ConfigureAwait(false); + + // 2. Load baseline + ctx.Status("Loading baseline seal..."); + FacetSeal? baseline; + if (!string.IsNullOrEmpty(baselineId)) + { + baseline = await sealStore.GetBySealIdAsync(baselineId, ct).ConfigureAwait(false); + } + else + { + baseline = await sealStore.GetLatestSealAsync(imageInfo.Digest, ct).ConfigureAwait(false); + } + + if (baseline is null) + { + AnsiConsole.MarkupLine("[red]No baseline seal found[/]"); + Environment.ExitCode = 1; + return; + } + + // 3. Compute drift + ctx.Status("Computing drift..."); + var report = await driftEngine.ComputeDriftAsync( + baseline, + imageInfo.FileSystem, + ct: ct).ConfigureAwait(false); + + // 4. Generate VEX + ctx.Status("Generating VEX statements..."); + var vexDocument = GenerateOpenVex(report, imageInfo.Digest, status, timeProvider, guidProvider); + + // 5. Output + var vexJson = JsonSerializer.Serialize(vexDocument, new JsonSerializerOptions { WriteIndented = true }); + + if (!string.IsNullOrEmpty(outputPath)) + { + await File.WriteAllTextAsync(outputPath, vexJson, ct).ConfigureAwait(false); + AnsiConsole.MarkupLine($"[green]VEX written to:[/] {outputPath}"); + } + else + { + Console.WriteLine(vexJson); + } + + AnsiConsole.MarkupLine($"[green]Generated {vexDocument.Statements.Length} VEX statement(s)[/]"); + }); + } + + private static OpenVexDocument GenerateOpenVex( + FacetDriftReport report, + string imageDigest, + string status, + TimeProvider timeProvider, + IGuidProvider guidProvider) + { + var now = timeProvider.GetUtcNow(); + var statements = new List(); + + foreach (var drift in report.FacetDrifts.Where(d => + d.QuotaVerdict == QuotaVerdict.RequiresVex || + d.QuotaVerdict == QuotaVerdict.Warning)) + { + statements.Add(new OpenVexStatement + { + Id = $"vex:{guidProvider.NewGuid()}", + Status = status, + Timestamp = now.ToString("O"), + Products = new[] + { + new OpenVexProduct + { + Id = imageDigest, + Identifiers = new { facet = drift.FacetId } + } + }, + Justification = $"Facet drift authorization for {drift.FacetId}. " + + $"Churn: {drift.ChurnPercent:F2}% " + + $"({drift.Added.Length} added, {drift.Removed.Length} removed, {drift.Modified.Length} modified)", + ActionStatement = drift.QuotaVerdict == QuotaVerdict.RequiresVex + ? "Review required before deployment" + : "Drift within acceptable limits" + }); + } + + return new OpenVexDocument + { + Context = "https://openvex.dev/ns", + Id = $"https://stellaops.io/vex/{guidProvider.NewGuid()}", + Author = "StellaOps CLI", + Timestamp = now.ToString("O"), + Version = 1, + Statements = [.. statements] + }; + } +} + +// OpenVEX document models +internal sealed record OpenVexDocument +{ + [JsonPropertyName("@context")] + public required string Context { get; init; } + + [JsonPropertyName("@id")] + public required string Id { get; init; } + + [JsonPropertyName("author")] + public required string Author { get; init; } + + [JsonPropertyName("timestamp")] + public required string Timestamp { get; init; } + + [JsonPropertyName("version")] + public required int Version { get; init; } + + [JsonPropertyName("statements")] + public required ImmutableArray Statements { get; init; } +} + +internal sealed record OpenVexStatement +{ + [JsonPropertyName("@id")] + public required string Id { get; init; } + + [JsonPropertyName("status")] + public required string Status { get; init; } + + [JsonPropertyName("timestamp")] + public required string Timestamp { get; init; } + + [JsonPropertyName("products")] + public required OpenVexProduct[] Products { get; init; } + + [JsonPropertyName("justification")] + public required string Justification { get; init; } + + [JsonPropertyName("action_statement")] + public string? ActionStatement { get; init; } +} + +internal sealed record OpenVexProduct +{ + [JsonPropertyName("@id")] + public required string Id { get; init; } + + [JsonPropertyName("identifiers")] + public required object Identifiers { get; init; } +} +``` + +### Admission Webhook Integration + +```csharp +// src/Zastava/StellaOps.Zastava.Webhook/Admission/FacetAdmissionValidator.cs +namespace StellaOps.Zastava.Webhook.Admission; + +/// +/// Validates facet seals during admission. +/// +public sealed class FacetAdmissionValidator : IAdmissionValidator +{ + private readonly IFacetSealStore _sealStore; + private readonly IFacetDriftEngine _driftEngine; + private readonly IDsseVerifier _dsseVerifier; + private readonly ILogger _logger; + + public string ValidatorId => "facet-seal"; + public int Priority => 60; // After policy, before budget + + public FacetAdmissionValidator( + IFacetSealStore sealStore, + IFacetDriftEngine driftEngine, + IDsseVerifier dsseVerifier, + ILogger? logger = null) + { + _sealStore = sealStore; + _driftEngine = driftEngine; + _dsseVerifier = dsseVerifier; + _logger = logger ?? NullLogger.Instance; + } + + public async Task ValidateAsync( + AdmissionRequest request, + CancellationToken ct = default) + { + // Check if facet validation is required for this namespace + if (!IsFacetValidationRequired(request)) + { + return AdmissionResult.Allow("Facet validation not required for namespace"); + } + + // Load seal for image + var seal = await _sealStore.GetLatestSealAsync(request.ImageDigest, ct) + .ConfigureAwait(false); + + if (seal is null) + { + if (request.NamespacePolicy.RequireSeal) + { + return AdmissionResult.Deny( + "facet.seal.missing", + $"No facet seal found for image {request.ImageDigest}"); + } + return AdmissionResult.Allow("No seal found, seal not required"); + } + + // Verify seal signature + if (seal.Signature is not null) + { + var sigResult = await _dsseVerifier.VerifyAsync(seal.Signature, ct) + .ConfigureAwait(false); + if (!sigResult.IsValid) + { + return AdmissionResult.Deny( + "facet.seal.invalid_signature", + "Facet seal signature verification failed"); + } + } + + // Compute drift + var driftReport = await _driftEngine.ComputeDriftAsync( + seal, + request.ImageFileSystem, + ct: ct).ConfigureAwait(false); + + _logger.LogInformation( + "Facet drift for {Image}: verdict={Verdict}, changes={Changes}", + request.ImageDigest, driftReport.OverallVerdict, driftReport.TotalChangedFiles); + + // Evaluate verdict + return driftReport.OverallVerdict switch + { + QuotaVerdict.Ok => AdmissionResult.Allow( + $"Facet quotas OK: {driftReport.TotalChangedFiles} changes within limits"), + + QuotaVerdict.Warning => AdmissionResult.AllowWithWarning( + "facet.quota.warning", + $"Facet drift warning: {FormatBreaches(driftReport)}"), + + QuotaVerdict.Blocked => AdmissionResult.Deny( + "facet.quota.exceeded", + $"Facet quota exceeded: {FormatBreaches(driftReport)}"), + + QuotaVerdict.RequiresVex => AdmissionResult.Deny( + "facet.vex.required", + $"Facet drift requires VEX authorization: {FormatBreaches(driftReport)}"), + + _ => AdmissionResult.Allow("Unknown verdict - allowing") + }; + } + + private static bool IsFacetValidationRequired(AdmissionRequest request) + { + // Check namespace annotation: stellaops.io/facet-seal-required=true + return request.NamespaceAnnotations.TryGetValue( + "stellaops.io/facet-seal-required", out var value) && + string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static string FormatBreaches(FacetDriftReport report) + { + return string.Join(", ", report.FacetDrifts + .Where(d => d.QuotaVerdict != QuotaVerdict.Ok) + .Select(d => $"{d.FacetId}({d.ChurnPercent:F1}%)")); + } +} +``` + +--- + +## Delivery Tracker + +| # | Task ID | Status | Dependency | Owners | Task Definition | +|---|---------|--------|------------|--------|-----------------| +| **stella seal Command** | +| 1 | CLI-001 | TODO | FCT models | CLI Guild | Create `SealCommandGroup.cs` with command structure | +| 2 | CLI-002 | TODO | CLI-001 | CLI Guild | Implement image resolution and layer extraction | +| 3 | CLI-003 | TODO | CLI-002 | CLI Guild | Implement facet extraction orchestration | +| 4 | CLI-004 | TODO | CLI-003 | CLI Guild | Implement DSSE signing integration | +| 5 | CLI-005 | TODO | CLI-004 | CLI Guild | Implement remote seal storage | +| 6 | CLI-006 | TODO | CLI-005 | CLI Guild | Unit tests: Seal command | +| **stella drift Command** | +| 7 | CLI-007 | TODO | QTA models | CLI Guild | Create `DriftCommandGroup.cs` | +| 8 | CLI-008 | TODO | CLI-007 | CLI Guild | Implement baseline loading | +| 9 | CLI-009 | TODO | CLI-008 | CLI Guild | Implement drift report formatting | +| 10 | CLI-010 | TODO | CLI-009 | CLI Guild | Unit tests: Drift command | +| **stella vex gen --from-drift Command** | +| 11 | CLI-011 | TODO | QTA models | CLI Guild | Create `VexGenCommandGroup.cs` | +| 12 | CLI-012 | TODO | CLI-011 | CLI Guild | Implement OpenVEX document generation | +| 13 | CLI-013 | TODO | CLI-012 | CLI Guild | Implement CSAF format (optional) | +| 14 | CLI-014 | TODO | CLI-013 | CLI Guild | Unit tests: VEX gen command | +| 15 | CLI-015 | TODO | CLI-014 | CLI Guild | Wire commands into main CLI | +| **Admission Integration** | +| 16 | ADM-001 | TODO | FCT, QTA | Zastava Guild | Create `FacetAdmissionValidator` class | +| 17 | ADM-002 | TODO | ADM-001 | Zastava Guild | Implement namespace annotation checking | +| 18 | ADM-003 | TODO | ADM-002 | Zastava Guild | Implement seal signature verification | +| 19 | ADM-004 | TODO | ADM-003 | Zastava Guild | Integrate with existing admission pipeline | +| 20 | ADM-005 | TODO | ADM-004 | Zastava Guild | Add admission metrics and logging | +| 21 | ADM-006 | TODO | ADM-005 | Zastava Guild | Unit tests: Admission validator | +| 22 | ADM-007 | TODO | ADM-006 | Zastava Guild | Integration tests: Full admission flow | +| **Documentation** | +| 23 | CLI-016 | TODO | CLI-015 | Docs Guild | Update `docs/cli/admin-reference.md` | +| 24 | CLI-017 | TODO | ADM-007 | Docs Guild | Document admission webhook configuration | +| 25 | CLI-018 | TODO | CLI-017 | QA Guild | E2E test: seal → deploy → drift check | + +--- + +## Success Metrics + +| Metric | Before | After | Target | +|--------|--------|-------|--------| +| `stella seal` command | No | Yes | Functional | +| `stella drift` command | No | Yes | Functional | +| `stella vex gen --from-drift` | No | Yes | Functional | +| Admission facet verification | No | Yes | Opt-in per namespace | +| <24h MTTR for unauthorized changes | N/A | Measured | <24h | + +--- + +## Execution Log + +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2026-01-05 | Sprint created from product advisory gap analysis | Planning | + +--- + +## Decisions & Risks + +| Decision/Risk | Type | Mitigation | +|---------------|------|------------| +| Image layer extraction adds latency | Trade-off | Cache extracted layers, parallelize | +| VEX format compatibility | Decision | Default to OpenVEX, CSAF optional | +| Admission webhook performance | Risk | Make facet validation opt-in via annotation | +| DSSE key distribution | Constraint | Document key management, use Sigstore | + +--- + +## Next Checkpoints + +- CLI-001 through CLI-006 (stella seal) target completion +- CLI-007 through CLI-010 (stella drift) target completion +- CLI-011 through CLI-015 (stella vex gen) target completion +- ADM-001 through ADM-007 (admission) target completion +- CLI-018 (E2E) sprint completion gate diff --git a/docs/interop/cosign-integration.md b/docs/interop/cosign-integration.md index e4864f802..ce6ba9a1c 100644 --- a/docs/interop/cosign-integration.md +++ b/docs/interop/cosign-integration.md @@ -612,8 +612,8 @@ stella attest verify-batch \ ### StellaOps Documentation - [Attestor Architecture](../modules/attestor/architecture.md) - [Standard Predicate Types](../modules/attestor/predicate-parsers.md) -- [CLI Reference](../09_API_CLI_REFERENCE.md) -- [Offline Kit Guide](../24_OFFLINE_KIT.md) +- [CLI Reference](../API_CLI_REFERENCE.md) +- [Offline Kit Guide](../OFFLINE_KIT.md) --- diff --git a/docs/market/claims-citation-index.md b/docs/market/claims-citation-index.md index 0b1f61325..9a2b02166 100644 --- a/docs/market/claims-citation-index.md +++ b/docs/market/claims-citation-index.md @@ -60,7 +60,7 @@ This document is the **authoritative source** for all competitive positioning cl |----|-------|----------|------------|----------|-------------| | PROOF-001 | "Deterministic proof ledgers with canonical JSON and CBOR serialization" | `docs/db/SPECIFICATION.md` Section 5.6-5.7 (policy.proof_segments, scanner.proof_bundle) | High | 2025-12-20 | 2026-03-20 | | PROOF-002 | "Cryptographic proof chains link scans to frozen feed state via Merkle roots" | `scanner.scan_manifest` (concelier_snapshot_hash, excititor_snapshot_hash) | High | 2025-12-20 | 2026-03-20 | -| PROOF-003 | "Score replay command verifies proof integrity against original calculation" | `stella score replay --scan --verify-proof`; `docs/24_OFFLINE_KIT.md` Section 2.2 | High | 2025-12-20 | 2026-03-20 | +| PROOF-003 | "Score replay command verifies proof integrity against original calculation" | `stella score replay --scan --verify-proof`; `docs/OFFLINE_KIT.md` Section 2.2 | High | 2025-12-20 | 2026-03-20 | ### 5. Offline & Air-Gap Claims diff --git a/docs/modules/README.md b/docs/modules/README.md index 666042170..722b7bbe1 100644 --- a/docs/modules/README.md +++ b/docs/modules/README.md @@ -20,7 +20,7 @@ This directory contains architecture documentation for all StellaOps modules. | [Concelier](./concelier/) | `src/Concelier/` | Vulnerability advisory ingestion and merge engine | | [Excititor](./excititor/) | `src/Excititor/` | VEX document ingestion and export | | [VexLens](./vex-lens/) | `src/VexLens/` | VEX consensus computation across issuers | -| [VexHub](./vexhub/) | `src/VexHub/` | VEX distribution and exchange hub | +| [VexHub](./vex-hub/) | `src/VexHub/` | VEX distribution and exchange hub | | [IssuerDirectory](./issuer-directory/) | `src/IssuerDirectory/` | Issuer trust registry (CSAF publishers) | | [Feedser](./feedser/) | `src/Feedser/` | Evidence collection library for backport detection | | [Mirror](./mirror/) | `src/Mirror/` | Vulnerability feed mirror and distribution | @@ -30,10 +30,10 @@ This directory contains architecture documentation for all StellaOps modules. | Module | Path | Description | |--------|------|-------------| | [Scanner](./scanner/) | `src/Scanner/` | Container scanning with SBOM generation | -| [BinaryIndex](./binaryindex/) | `src/BinaryIndex/` | Binary identity extraction and fingerprinting | +| [BinaryIndex](./binary-index/) | `src/BinaryIndex/` | Binary identity extraction and fingerprinting | | [AdvisoryAI](./advisory-ai/) | `src/AdvisoryAI/` | AI-assisted advisory analysis | | [Symbols](./symbols/) | `src/Symbols/` | Symbol resolution and debug information | -| [ReachGraph](./reachgraph/) | `src/ReachGraph/` | Reachability graph service | +| [ReachGraph](./reach-graph/) | `src/ReachGraph/` | Reachability graph service | ### Artifacts & Evidence @@ -41,18 +41,18 @@ This directory contains architecture documentation for all StellaOps modules. |--------|------|-------------| | [Attestor](./attestor/) | `src/Attestor/` | in-toto/DSSE attestation generation | | [Signer](./signer/) | `src/Signer/` | Cryptographic signing operations | -| [SbomService](./sbomservice/) | `src/SbomService/` | SBOM storage, versioning, and lineage ledger | +| [SbomService](./sbom-service/) | `src/SbomService/` | SBOM storage, versioning, and lineage ledger | | [EvidenceLocker](./evidence-locker/) | `src/EvidenceLocker/` | Sealed evidence storage and export | | [ExportCenter](./export-center/) | `src/ExportCenter/` | Batch export and report generation | | [Provenance](./provenance/) | `src/Provenance/` | SLSA/DSSE attestation tooling | -| [Provcache](./provcache/) | Library | Provenance cache utilities | +| [Provcache](./prov-cache/) | Library | Provenance cache utilities | ### Policy & Risk | Module | Path | Description | |--------|------|-------------| | [Policy](./policy/) | `src/Policy/` | Policy engine with K4 lattice logic | -| [RiskEngine](./riskengine/) | `src/RiskEngine/` | Risk scoring runtime | +| [RiskEngine](./risk-engine/) | `src/RiskEngine/` | Risk scoring runtime | | [VulnExplorer](./vuln-explorer/) | `src/VulnExplorer/` | Vulnerability exploration and triage | | [Unknowns](./unknowns/) | `src/Unknowns/` | Unknown component tracking registry | @@ -65,8 +65,8 @@ This directory contains architecture documentation for all StellaOps modules. | [TaskRunner](./taskrunner/) | `src/TaskRunner/` | Task pack execution engine | | [Notify](./notify/) | `src/Notify/` | Notification toolkit (Email, Slack, Teams, Webhooks) | | [Notifier](./notifier/) | `src/Notifier/` | Notifications Studio host | -| [PacksRegistry](./packsregistry/) | `src/PacksRegistry/` | Task packs registry | -| [TimelineIndexer](./timelineindexer/) | `src/TimelineIndexer/` | Timeline event indexing | +| [PacksRegistry](./packs-registry/) | `src/PacksRegistry/` | Task packs registry | +| [TimelineIndexer](./timeline-indexer/) | `src/TimelineIndexer/` | Timeline event indexing | | [Replay](./replay/) | `src/Replay/` | Deterministic replay engine | ### Integration diff --git a/docs/modules/advisory-ai/guides/offline-model-bundles.md b/docs/modules/advisory-ai/guides/offline-model-bundles.md index 08b7019fe..9d19a0295 100644 --- a/docs/modules/advisory-ai/guides/offline-model-bundles.md +++ b/docs/modules/advisory-ai/guides/offline-model-bundles.md @@ -273,6 +273,6 @@ stella model benchmark llama3-8b-q4km --iterations 10 ## Related Documentation - [Advisory AI Architecture](../architecture.md) -- [Offline Kit Overview](../../../24_OFFLINE_KIT.md) +- [Offline Kit Overview](../../../OFFLINE_KIT.md) - [AI Attestations](../../../implplan/SPRINT_20251226_018_AI_attestations.md) - [Replay Semantics](./replay-semantics.md) diff --git a/docs/modules/airgap/README.md b/docs/modules/airgap/README.md index fd9669406..041d4b8d6 100644 --- a/docs/modules/airgap/README.md +++ b/docs/modules/airgap/README.md @@ -42,7 +42,7 @@ Key settings: ## Related Documentation - Operations: `./operations/` (if exists) -- Offline Kit: `../../24_OFFLINE_KIT.md` +- Offline Kit: `../../OFFLINE_KIT.md` - Mirror: `../mirror/` - ExportCenter: `../export-center/` diff --git a/docs/modules/airgap/architecture.md b/docs/modules/airgap/architecture.md index 7f35511fa..ad43188f7 100644 --- a/docs/modules/airgap/architecture.md +++ b/docs/modules/airgap/architecture.md @@ -344,5 +344,5 @@ AirGap: * Evidence reconciliation: `./evidence-reconciliation.md` * Exporter coordination: `./exporter-cli-coordination.md` * Mirror DSSE plan: `./mirror-dsse-plan.md` -* Offline Kit: `../../24_OFFLINE_KIT.md` +* Offline Kit: `../../OFFLINE_KIT.md` * Time anchor schema: `../../airgap/time-anchor-schema.md` diff --git a/docs/modules/authority/verdict-manifest.md b/docs/modules/authority/verdict-manifest.md index 4635f54c3..8dc5b299a 100644 --- a/docs/modules/authority/verdict-manifest.md +++ b/docs/modules/authority/verdict-manifest.md @@ -489,7 +489,7 @@ Content-Disposition: attachment; filename="verdict-{manifestId}.json" - [Trust Lattice Specification](../excititor/trust-lattice.md) - [Authority Architecture](./architecture.md) - [DSSE Signing](../../dev/dsse-signing.md) -- [API Reference](../../09_API_CLI_REFERENCE.md) +- [API Reference](../../API_CLI_REFERENCE.md) --- diff --git a/docs/modules/binary-index/README.md b/docs/modules/binary-index/README.md new file mode 100644 index 000000000..320e84b05 --- /dev/null +++ b/docs/modules/binary-index/README.md @@ -0,0 +1,94 @@ +# BinaryIndex + +**Status:** Implemented +**Source:** `src/BinaryIndex/` +**Owner:** Scanner Guild + Concelier Guild + +## Purpose + +BinaryIndex provides vulnerable binary detection independent of package metadata. It addresses the gap where package version strings can lie (backports, custom builds, stripped metadata) through binary-first vulnerability identification using Build-IDs, hash catalogs, and function fingerprints. + +## Components + +**Libraries:** +- `StellaOps.BinaryIndex.Core` - Core binary identity extraction and matching engine +- `StellaOps.BinaryIndex.Corpus` - Binary-to-advisory mapping database +- `StellaOps.BinaryIndex.Corpus.Debian` - Debian-specific corpus support +- `StellaOps.BinaryIndex.Fingerprints` - Function fingerprint storage and matching (CFG/basic-block hashes) +- `StellaOps.BinaryIndex.FixIndex` - Patch-aware backport handling +- `StellaOps.BinaryIndex.Persistence` - Storage adapters for binary catalogs + +## Configuration + +Configuration is typically embedded in Scanner and Concelier module settings. + +Key features: +- Three-tier binary identification (package/version, Build-ID/hash, function fingerprints) +- Binary identity extraction (Build-ID, PE CodeView GUID, Mach-O UUID) +- Integration with Scanner.Worker for binary lookup +- Offline-first design with deterministic outputs + +## Dependencies + +- PostgreSQL (integrated with Scanner/Concelier schemas) +- Scanner.Analyzers.Native (for binary disassembly/analysis) +- Concelier (for advisory-to-binary mapping) + +## Related Documentation + +- Architecture: `./architecture.md` +- High-Level Architecture: `../../ARCHITECTURE_OVERVIEW.md` +- Scanner Architecture: `../scanner/architecture.md` +- Concelier Architecture: `../concelier/architecture.md` + +## Current Status + +Library implementation complete with support for ELF (Build-ID), PE (CodeView GUID), and Mach-O (UUID) binary formats. Integrated into Scanner's native binary analysis pipeline. + +--- + +## Semantic Diffing Roadmap + +A major enhancement to BinaryIndex is planned to enable **semantic-level binary diffing** - detecting function equivalence based on behavior rather than syntax. This addresses limitations in current byte/symbol-based matching when dealing with: + +- Compiler optimizations (same source, different instructions) +- Stripped binaries (no symbols) +- Cross-compiler builds (GCC vs Clang) +- Obfuscated code + +### Planned Phases + +| Phase | Description | Impact | Status | +|-------|-------------|--------|--------| +| **Phase 1** | IR-Level Semantic Analysis | +15% accuracy on optimized binaries | Planned | +| **Phase 2** | Function Behavior Corpus | +10% coverage on stripped binaries | Planned | +| **Phase 3** | Ghidra Integration | +5% edge case handling | Planned | +| **Phase 4** | Decompiler & ML Similarity | +10% obfuscation resilience | Planned | + +### New Libraries (Planned) + +- `StellaOps.BinaryIndex.Semantic` - IR lifting and semantic graph fingerprints +- `StellaOps.BinaryIndex.Corpus` - 30K+ function behavior database +- `StellaOps.BinaryIndex.Ghidra` - Ghidra Headless integration +- `StellaOps.BinaryIndex.Decompiler` - Decompiled code AST comparison +- `StellaOps.BinaryIndex.ML` - CodeBERT-based function embeddings +- `StellaOps.BinaryIndex.Ensemble` - Multi-signal decision fusion + +### Expected Outcomes + +| Metric | Current | Target | +|--------|---------|--------| +| Patch detection accuracy | ~70% | 92%+ | +| Function identification (stripped) | ~50% | 85%+ | +| False positive rate | ~5% | <2% | + +### Sprint Files + +- `docs/implplan/SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md` +- `docs/implplan/SPRINT_20260105_001_002_BINDEX_semdiff_corpus.md` +- `docs/implplan/SPRINT_20260105_001_003_BINDEX_semdiff_ghidra.md` +- `docs/implplan/SPRINT_20260105_001_004_BINDEX_semdiff_decompiler_ml.md` + +### Architecture Documentation + +See `./semantic-diffing.md` for comprehensive architecture documentation. diff --git a/docs/modules/binaryindex/architecture.md b/docs/modules/binary-index/architecture.md similarity index 99% rename from docs/modules/binaryindex/architecture.md rename to docs/modules/binary-index/architecture.md index 3cad6917a..477a84d47 100644 --- a/docs/modules/binaryindex/architecture.md +++ b/docs/modules/binary-index/architecture.md @@ -3,7 +3,7 @@ > **Ownership:** Scanner Guild + Concelier Guild > **Status:** DRAFT > **Version:** 1.0.0 -> **Related:** [High-Level Architecture](../../07_HIGH_LEVEL_ARCHITECTURE.md), [Scanner Architecture](../scanner/architecture.md), [Concelier Architecture](../concelier/architecture.md) +> **Related:** [High-Level Architecture](../../ARCHITECTURE_OVERVIEW.md), [Scanner Architecture](../scanner/architecture.md), [Concelier Architecture](../concelier/architecture.md) --- diff --git a/docs/modules/binary-index/semantic-diffing.md b/docs/modules/binary-index/semantic-diffing.md new file mode 100644 index 000000000..2f7a1d832 --- /dev/null +++ b/docs/modules/binary-index/semantic-diffing.md @@ -0,0 +1,564 @@ +# Semantic Diffing Architecture + +> **Status:** PLANNED +> **Version:** 1.0.0 +> **Related Sprints:** +> - `SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md` +> - `SPRINT_20260105_001_002_BINDEX_semdiff_corpus.md` +> - `SPRINT_20260105_001_003_BINDEX_semdiff_ghidra.md` +> - `SPRINT_20260105_001_004_BINDEX_semdiff_decompiler_ml.md` + +--- + +## 1. Executive Summary + +Semantic diffing is an advanced binary analysis capability that detects function equivalence based on **behavior** rather than **syntax**. This enables accurate vulnerability detection in scenarios where traditional byte-level or symbol-based matching fails: + +- **Compiler optimizations** - Same source, different instructions +- **Obfuscation** - Intentionally altered code structure +- **Stripped binaries** - No symbols or debug information +- **Cross-compiler** - GCC vs Clang produce different output +- **Backported patches** - Different version, same fix + +### Expected Impact + +| Capability | Current Accuracy | With Semantic Diffing | +|------------|-----------------|----------------------| +| Patch detection (optimized) | ~70% | 92%+ | +| Function identification (stripped) | ~50% | 85%+ | +| Obfuscation resilience | ~40% | 75%+ | +| False positive rate | ~5% | <2% | + +--- + +## 2. Architecture Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────────┐ +│ Semantic Diffing Architecture │ +│ │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐│ +│ │ Analysis Layer ││ +│ │ ││ +│ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ││ +│ │ │ B2R2 │ │ Ghidra │ │ Decompiler │ │ ML │ ││ +│ │ │ (Primary) │ │ (Fallback) │ │ (Optional) │ │ (Optional) │ ││ +│ │ │ │ │ │ │ │ │ │ ││ +│ │ │ - Disasm │ │ - P-Code │ │ - C output │ │ - CodeBERT │ ││ +│ │ │ - LowUIR │ │ - BSim │ │ - AST parse │ │ - GraphSage │ ││ +│ │ │ - CFG │ │ - Ver.Track │ │ - Normalize │ │ - Embedding │ ││ +│ │ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ ││ +│ │ │ │ │ │ ││ +│ └─────────┴────────────────┴────────────────┴────────────────┴───────────────┘│ +│ │ │ +│ v │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐│ +│ │ Fingerprint Layer ││ +│ │ ││ +│ │ ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ ││ +│ │ │ Instruction │ │ Semantic │ │ Decompiled │ ││ +│ │ │ Fingerprint │ │ Fingerprint │ │ Fingerprint │ ││ +│ │ │ │ │ │ │ │ ││ +│ │ │ - BasicBlock hash │ │ - KSG graph hash │ │ - AST hash │ ││ +│ │ │ - CFG edge hash │ │ - WL hash │ │ - Normalized code │ ││ +│ │ │ - String refs │ │ - DataFlow hash │ │ - API sequence │ ││ +│ │ │ - Rolling chunks │ │ - API calls │ │ - Pattern hash │ ││ +│ │ └───────────────────┘ └───────────────────┘ └───────────────────┘ ││ +│ │ ││ +│ │ ┌───────────────────┐ ┌───────────────────┐ ││ +│ │ │ BSim │ │ ML Embedding │ ││ +│ │ │ Signature │ │ Vector │ ││ +│ │ │ │ │ │ ││ +│ │ │ - Feature vector │ │ - 768-dim float[] │ ││ +│ │ │ - Significance │ │ - Cosine sim │ ││ +│ │ └───────────────────┘ └───────────────────┘ ││ +│ │ ││ +│ └─────────────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐│ +│ │ Matching Layer ││ +│ │ ││ +│ │ ┌───────────────────────────────────────────────────────────────────────┐ ││ +│ │ │ Ensemble Decision Engine │ ││ +│ │ │ │ ││ +│ │ │ Signal Weights: │ ││ +│ │ │ - Instruction fingerprint: 15% │ ││ +│ │ │ - Semantic graph: 25% │ ││ +│ │ │ - Decompiled AST: 35% │ ││ +│ │ │ - ML embedding: 25% │ ││ +│ │ │ │ ││ +│ │ │ Output: Confidence-weighted similarity score │ ││ +│ │ │ │ ││ +│ │ └───────────────────────────────────────────────────────────────────────┘ ││ +│ │ ││ +│ └─────────────────────────────────────────────────────────────────────────────┘│ +│ │ │ +│ v │ +│ ┌─────────────────────────────────────────────────────────────────────────────┐│ +│ │ Storage Layer ││ +│ │ ││ +│ │ PostgreSQL RustFS Valkey ││ +│ │ - corpus.* tables - Fingerprint blobs - Query cache ││ +│ │ - binaries.* tables - Model artifacts - Embedding index ││ +│ │ - BSim database - Training data ││ +│ │ ││ +│ └─────────────────────────────────────────────────────────────────────────────┘│ +└─────────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 3. Implementation Phases + +### Phase 1: IR-Level Semantic Analysis (Foundation) + +**Sprint:** `SPRINT_20260105_001_001_BINDEX_semdiff_ir_semantics.md` + +Leverage B2R2's Intermediate Representation (IR) for semantic-level function comparison. + +**Key Components:** +- `IrLiftingService` - Lift instructions to LowUIR +- `SemanticGraphExtractor` - Build Key-Semantics Graph (KSG) +- `WeisfeilerLehmanHasher` - Graph fingerprinting +- `SemanticMatcher` - Semantic similarity scoring + +**Deliverables:** +- `StellaOps.BinaryIndex.Semantic` library +- 20 tasks, ~3 weeks + +### Phase 2: Function Behavior Corpus (Scale) + +**Sprint:** `SPRINT_20260105_001_002_BINDEX_semdiff_corpus.md` + +Build comprehensive database of known library functions. + +**Key Components:** +- Library corpus connectors (glibc, OpenSSL, zlib, curl, SQLite) +- `CorpusIngestionService` - Batch fingerprint generation +- `FunctionClusteringService` - Group similar functions +- `CorpusQueryService` - Function identification + +**Deliverables:** +- `StellaOps.BinaryIndex.Corpus` library +- PostgreSQL `corpus.*` schema +- ~30,000 indexed functions +- 22 tasks, ~4 weeks + +### Phase 3: Ghidra Integration (Depth) + +**Sprint:** `SPRINT_20260105_001_003_BINDEX_semdiff_ghidra.md` + +Add Ghidra as secondary backend for complex cases. + +**Key Components:** +- `GhidraHeadlessManager` - Process lifecycle +- `VersionTrackingService` - Multi-correlator diffing +- `GhidriffBridge` - Python interop +- `BSimService` - Behavioral similarity + +**Deliverables:** +- `StellaOps.BinaryIndex.Ghidra` library +- Docker image for Ghidra Headless +- 20 tasks, ~4 weeks + +### Phase 4: Decompiler & ML (Excellence) + +**Sprint:** `SPRINT_20260105_001_004_BINDEX_semdiff_decompiler_ml.md` + +Highest-fidelity semantic analysis. + +**Key Components:** +- `IDecompilerService` - Ghidra decompilation +- `AstComparisonEngine` - Structural similarity +- `OnnxInferenceEngine` - ML embeddings +- `EnsembleDecisionEngine` - Multi-signal fusion + +**Deliverables:** +- `StellaOps.BinaryIndex.Decompiler` library +- `StellaOps.BinaryIndex.ML` library +- Trained CodeBERT-Binary model +- 30 tasks, ~5 weeks + +--- + +## 4. Fingerprint Types + +### 4.1 Instruction Fingerprint (Existing) + +**Algorithm:** BasicBlock hash + CFG edge hash + String refs hash + +**Properties:** +- Fast to compute +- Sensitive to instruction changes +- Good for exact/near-exact matches + +**Weight in ensemble:** 15% + +### 4.2 Semantic Fingerprint (Phase 1) + +**Algorithm:** Key-Semantics Graph + Weisfeiler-Lehman hash + +**Properties:** +- Captures data/control dependencies +- Resilient to register renaming +- Resilient to instruction reordering + +**Weight in ensemble:** 25% + +### 4.3 Decompiled Fingerprint (Phase 4) + +**Algorithm:** Normalized AST hash + Pattern detection + +**Properties:** +- Highest semantic fidelity +- Captures algorithmic structure +- Resilient to most optimizations + +**Weight in ensemble:** 35% + +### 4.4 ML Embedding (Phase 4) + +**Algorithm:** CodeBERT-Binary transformer, 768-dim vectors + +**Properties:** +- Learned similarity metric +- Captures latent patterns +- Resilient to obfuscation + +**Weight in ensemble:** 25% + +--- + +## 5. Matching Pipeline + +```mermaid +sequenceDiagram + participant Client + participant DiffEngine as PatchDiffEngine + participant B2R2 + participant Ghidra + participant Corpus + participant Ensemble + + Client->>DiffEngine: Compare(oldBinary, newBinary) + + par Parallel Analysis + DiffEngine->>B2R2: Disassemble + IR lift + DiffEngine->>Ghidra: Decompile (if needed) + end + + B2R2-->>DiffEngine: SemanticFingerprints[] + Ghidra-->>DiffEngine: DecompiledFunctions[] + + DiffEngine->>Corpus: IdentifyFunctions(fingerprints) + Corpus-->>DiffEngine: FunctionMatches[] + + DiffEngine->>Ensemble: ComputeSimilarity(old, new) + Ensemble-->>DiffEngine: EnsembleResult + + DiffEngine-->>Client: PatchDiffResult +``` + +--- + +## 6. Fallback Strategy + +The system uses a tiered fallback strategy: + +``` +Tier 1: B2R2 IR + Semantic Graph (fast, ~90% coverage) + │ + │ If confidence < threshold OR architecture unsupported + v +Tier 2: Ghidra Version Tracking (slower, ~95% coverage) + │ + │ If function is high-value (CVE-relevant) + v +Tier 3: Decompiled AST + ML Embedding (slowest, ~99% coverage) +``` + +**Selection Criteria:** + +| Condition | Backend | Reason | +|-----------|---------|--------| +| Standard x64/ARM64 binary | B2R2 only | Fast, accurate | +| Low B2R2 confidence (<0.7) | B2R2 + Ghidra | Validation | +| Exotic architecture | Ghidra only | Better coverage | +| CVE-affected function | Full pipeline | Maximum accuracy | +| Obfuscated binary | ML embedding | Obfuscation resilience | + +--- + +## 7. Corpus Coverage + +### Priority Libraries + +| Library | Priority | Functions | CVEs | +|---------|----------|-----------|------| +| glibc | Critical | ~15,000 | 50+ | +| OpenSSL | Critical | ~8,000 | 100+ | +| zlib | High | ~200 | 5+ | +| libcurl | High | ~2,000 | 80+ | +| SQLite | High | ~1,500 | 30+ | +| libxml2 | Medium | ~1,200 | 40+ | +| libpng | Medium | ~300 | 10+ | +| expat | Medium | ~150 | 15+ | + +### Architecture Coverage + +| Architecture | B2R2 | Ghidra | Status | +|--------------|------|--------|--------| +| x86_64 | Excellent | Excellent | Primary | +| ARM64 | Excellent | Excellent | Primary | +| ARM32 | Good | Excellent | Secondary | +| MIPS32 | Fair | Excellent | Fallback | +| MIPS64 | Fair | Excellent | Fallback | +| RISC-V | Good | Good | Emerging | +| PPC32/64 | Fair | Excellent | Fallback | + +--- + +## 8. Performance Characteristics + +### Latency Budget + +| Operation | Target | Notes | +|-----------|--------|-------| +| B2R2 disassembly | <100ms | Per function | +| IR lifting | <50ms | Per function | +| Semantic fingerprint | <50ms | Per function | +| Ghidra analysis | <30s | Per binary (startup) | +| Decompilation | <500ms | Per function | +| ML inference | <100ms | Per function | +| Ensemble decision | <10ms | Per comparison | +| **Total (Tier 1)** | **<200ms** | Per function | +| **Total (Full)** | **<1s** | Per function | + +### Memory Budget + +| Component | Memory | Notes | +|-----------|--------|-------| +| B2R2 per binary | ~100MB | Scales with binary size | +| Ghidra per project | ~2GB | Persistent cache | +| ML model | ~500MB | ONNX loaded | +| Corpus query cache | ~100MB | LRU eviction | + +--- + +## 9. Integration Points + +### 9.1 Scanner Integration + +```csharp +// Scanner.Worker uses semantic diffing for binary vulnerability detection +var result = await _binaryVulnerabilityService.LookupByFingerprintAsync( + fingerprint, + minSimilarity: 0.85m, + useSemanticMatching: true, // Enable semantic diffing + ct); +``` + +### 9.2 PatchDiffEngine Enhancement + +```csharp +// PatchDiffEngine now includes semantic comparison +var diff = await _patchDiffEngine.DiffAsync( + vulnerableBinary, + patchedBinary, + new PatchDiffOptions + { + UseSemanticAnalysis = true, + SemanticThreshold = 0.7m, + IncludeDecompilation = true, + IncludeMlEmbedding = true + }, + ct); +``` + +### 9.3 DeltaSignature Enhancement + +```csharp +// Delta signatures now include semantic fingerprints +var signature = await _deltaSignatureGenerator.GenerateSignaturesAsync( + binaryStream, + new DeltaSignatureRequest + { + Cve = "CVE-2024-1234", + TargetSymbols = ["vulnerable_func"], + IncludeSemanticFingerprint = true, + IncludeDecompiledHash = true + }, + ct); +``` + +--- + +## 10. Security Considerations + +### 10.1 Sandbox Requirements + +All binary analysis runs in sandboxed environments: +- Seccomp profile restricting syscalls +- Read-only root filesystem +- No network access during analysis +- Memory/CPU limits + +### 10.2 Model Security + +ML models are: +- Signed with DSSE attestations +- Verified before loading +- Not user-uploadable (pre-trained only) + +### 10.3 Corpus Integrity + +Corpus data is: +- Ingested from trusted sources only +- Signed at snapshot level +- Version-controlled with audit trail + +--- + +## 11. Configuration + +```yaml +# binaryindex.yaml - Semantic diffing configuration +binaryindex: + semantic_diffing: + enabled: true + + # Analysis backends + backends: + b2r2: + enabled: true + ir_lifting: true + semantic_graph: true + ghidra: + enabled: true + fallback_only: true + min_b2r2_confidence: 0.7 + headless_timeout_ms: 30000 + decompiler: + enabled: true + high_value_only: true # Only for CVE-affected functions + ml: + enabled: true + model_path: /models/codebert_binary_v1.onnx + embedding_dimension: 768 + + # Ensemble weights + ensemble: + instruction_weight: 0.15 + semantic_weight: 0.25 + decompiled_weight: 0.35 + ml_weight: 0.25 + min_confidence: 0.6 + + # Corpus + corpus: + auto_update: true + update_interval_hours: 24 + libraries: + - glibc + - openssl + - zlib + - curl + - sqlite + + # Performance + performance: + max_parallel_analyses: 4 + cache_ttl_seconds: 3600 + max_function_size_bytes: 1048576 # 1MB +``` + +--- + +## 12. Metrics & Observability + +### Metrics + +| Metric | Type | Labels | +|--------|------|--------| +| `semantic_diffing_analysis_total` | Counter | backend, result | +| `semantic_diffing_latency_ms` | Histogram | backend, tier | +| `semantic_diffing_accuracy` | Gauge | comparison_type | +| `corpus_functions_total` | Gauge | library | +| `ml_inference_latency_ms` | Histogram | model | +| `ensemble_signal_weight` | Gauge | signal_type | + +### Traces + +- `semantic_diffing.analyze` - Full analysis span +- `semantic_diffing.b2r2.lift` - IR lifting +- `semantic_diffing.ghidra.decompile` - Decompilation +- `semantic_diffing.ml.inference` - ML embedding +- `semantic_diffing.ensemble.decide` - Ensemble decision + +--- + +## 13. Testing Strategy + +### Unit Tests + +| Test Suite | Coverage | +|------------|----------| +| `IrLiftingServiceTests` | IR lifting correctness | +| `SemanticGraphExtractorTests` | Graph construction | +| `WeisfeilerLehmanHasherTests` | Hash stability | +| `AstComparisonEngineTests` | AST similarity | +| `OnnxInferenceEngineTests` | ML inference | +| `EnsembleDecisionEngineTests` | Weight combination | + +### Integration Tests + +| Test Suite | Coverage | +|------------|----------| +| `EndToEndSemanticDiffTests` | Full pipeline | +| `OptimizationResilienceTests` | O0 vs O2 vs O3 | +| `CompilerVariantTests` | GCC vs Clang | +| `GhidraFallbackTests` | Fallback scenarios | + +### Golden Corpus Tests + +Pre-computed test cases with known results: +- 100 CVE patch pairs (vulnerable -> fixed) +- 50 optimization variant sets +- 25 compiler variant sets +- 25 obfuscation variant sets + +--- + +## 14. Roadmap + +| Phase | Status | ETA | Impact | +|-------|--------|-----|--------| +| Phase 1: IR Semantics | Planned | 2026-01-24 | +15% accuracy | +| Phase 2: Corpus | Planned | 2026-02-15 | +10% coverage | +| Phase 3: Ghidra | Planned | 2026-02-28 | +5% edge cases | +| Phase 4: Decompiler/ML | Planned | 2026-03-31 | +10% obfuscation | +| **Total** | | | **+35-40%** | + +--- + +## 15. References + +### Internal + +- `docs/modules/binary-index/architecture.md` +- `src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.DeltaSig/` +- `src/BinaryIndex/__Libraries/StellaOps.BinaryIndex.Fingerprints/` + +### External + +- [B2R2 Binary Analysis Framework](https://b2r2.org/) +- [Ghidra Patch Diffing Guide](https://cve-north-stars.github.io/docs/Ghidra-Patch-Diffing) +- [ghidriff Tool](https://github.com/clearbluejar/ghidriff) +- [SemDiff Paper (arXiv)](https://arxiv.org/abs/2308.01463) +- [SEI Semantic Equivalence Research](https://www.sei.cmu.edu/annual-reviews/2022-research-review/semantic-equivalence-checking-of-decompiled-binaries/) + +--- + +*Document Version: 1.0.0* +*Last Updated: 2026-01-05* diff --git a/docs/modules/binaryindex/README.md b/docs/modules/binaryindex/README.md deleted file mode 100644 index 457d0cf52..000000000 --- a/docs/modules/binaryindex/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# BinaryIndex - -**Status:** Implemented -**Source:** `src/BinaryIndex/` -**Owner:** Scanner Guild + Concelier Guild - -## Purpose - -BinaryIndex provides vulnerable binary detection independent of package metadata. It addresses the gap where package version strings can lie (backports, custom builds, stripped metadata) through binary-first vulnerability identification using Build-IDs, hash catalogs, and function fingerprints. - -## Components - -**Libraries:** -- `StellaOps.BinaryIndex.Core` - Core binary identity extraction and matching engine -- `StellaOps.BinaryIndex.Corpus` - Binary-to-advisory mapping database -- `StellaOps.BinaryIndex.Corpus.Debian` - Debian-specific corpus support -- `StellaOps.BinaryIndex.Fingerprints` - Function fingerprint storage and matching (CFG/basic-block hashes) -- `StellaOps.BinaryIndex.FixIndex` - Patch-aware backport handling -- `StellaOps.BinaryIndex.Persistence` - Storage adapters for binary catalogs - -## Configuration - -Configuration is typically embedded in Scanner and Concelier module settings. - -Key features: -- Three-tier binary identification (package/version, Build-ID/hash, function fingerprints) -- Binary identity extraction (Build-ID, PE CodeView GUID, Mach-O UUID) -- Integration with Scanner.Worker for binary lookup -- Offline-first design with deterministic outputs - -## Dependencies - -- PostgreSQL (integrated with Scanner/Concelier schemas) -- Scanner.Analyzers.Native (for binary disassembly/analysis) -- Concelier (for advisory-to-binary mapping) - -## Related Documentation - -- Architecture: `./architecture.md` -- High-Level Architecture: `../../07_HIGH_LEVEL_ARCHITECTURE.md` -- Scanner Architecture: `../scanner/architecture.md` -- Concelier Architecture: `../concelier/architecture.md` - -## Current Status - -Library implementation complete with support for ELF (Build-ID), PE (CodeView GUID), and Mach-O (UUID) binary formats. Integrated into Scanner's native binary analysis pipeline. diff --git a/docs/modules/cli/guides/cli-reference.md b/docs/modules/cli/guides/cli-reference.md index 0292048a5..96f2be392 100644 --- a/docs/modules/cli/guides/cli-reference.md +++ b/docs/modules/cli/guides/cli-reference.md @@ -418,7 +418,7 @@ Additional notes: - [Aggregation-Only Contract reference](../../../aoc/aggregation-only-contract.md) - [Architecture overview](../../platform/architecture-overview.md) -- [Console operator guide](../../../15_UI_GUIDE.md) +- [Console operator guide](../../../UI_GUIDE.md) - [Authority scopes](../../authority/architecture.md) - [Task Pack CLI profiles](./packs-profiles.md) diff --git a/docs/modules/cli/guides/commands/scan-replay.md b/docs/modules/cli/guides/commands/scan-replay.md index 5bca55626..da3649e14 100644 --- a/docs/modules/cli/guides/commands/scan-replay.md +++ b/docs/modules/cli/guides/commands/scan-replay.md @@ -158,5 +158,5 @@ stella scan replay \ ## See Also - [Deterministic Replay Specification](../../replay/DETERMINISTIC_REPLAY.md) -- [Offline Kit Documentation](../../24_OFFLINE_KIT.md) +- [Offline Kit Documentation](../../OFFLINE_KIT.md) - [Evidence Bundle Format](./evidence-bundle-format.md) diff --git a/docs/modules/cli/guides/vuln-explorer-cli.md b/docs/modules/cli/guides/vuln-explorer-cli.md index 667585339..8bb347269 100644 --- a/docs/modules/cli/guides/vuln-explorer-cli.md +++ b/docs/modules/cli/guides/vuln-explorer-cli.md @@ -490,7 +490,7 @@ When operating in air-gapped environments: --expected-digest sha256:... ``` -For full offline kit support, see the [Offline Kit documentation](../../../24_OFFLINE_KIT.md). +For full offline kit support, see the [Offline Kit documentation](../../../OFFLINE_KIT.md). --- @@ -499,4 +499,4 @@ For full offline kit support, see the [Offline Kit documentation](../../../24_OF - [VEX Consensus CLI](./vex-cli.md) - VEX status management - [Policy Simulation](../../policy/guides/simulation.md) - Policy testing - [Authentication Guide](./auth-cli.md) - Token management -- [API Reference](../../../09_API_CLI_REFERENCE.md) - Full API documentation +- [API Reference](../../../API_CLI_REFERENCE.md) - Full API documentation diff --git a/docs/modules/concelier/README.md b/docs/modules/concelier/README.md index 99c94f700..ccbf92bae 100644 --- a/docs/modules/concelier/README.md +++ b/docs/modules/concelier/README.md @@ -35,13 +35,13 @@ Concelier ingests signed advisories from **32 advisory connectors** and converts - Connector runbooks in ./operations/connectors/. - Mirror operations for Offline Kit parity. - Grafana dashboards for connector health. -- **Authority toggle rollout (2025-10-22 update).** Follow the phased table and audit checklist in `../../10_CONCELIER_CLI_QUICKSTART.md` when enabling `authority.enabled`/`authority.allowAnonymousFallback`, and cross-check the refreshed `./operations/authority-audit-runbook.md` before enforcement. +- **Authority toggle rollout (2025-10-22 update).** Follow the phased table and audit checklist in `../../CONCELIER_CLI_QUICKSTART.md` when enabling `authority.enabled`/`authority.allowAnonymousFallback`, and cross-check the refreshed `./operations/authority-audit-runbook.md` before enforcement. ## Related resources - ./operations/conflict-resolution.md - ./operations/mirror.md - ./operations/authority-audit-runbook.md -- ../../10_CONCELIER_CLI_QUICKSTART.md (authority integration timeline & smoke tests) +- ../../CONCELIER_CLI_QUICKSTART.md (authority integration timeline & smoke tests) ## Backlog references - DOCS-LNM-22-001, DOCS-LNM-22-007 in ../../TASKS.md. diff --git a/docs/modules/concelier/operations/connectors/certbund.md b/docs/modules/concelier/operations/connectors/certbund.md index 0e22400aa..46ae5fa90 100644 --- a/docs/modules/concelier/operations/connectors/certbund.md +++ b/docs/modules/concelier/operations/connectors/certbund.md @@ -132,7 +132,7 @@ operating offline. ## 4. Locale & Translation Guidance - Advisories remain in German (`language: "de"`). Preserve wording for provenance and legal accuracy. -- UI localisation: enable the translation bundles documented in `docs/15_UI_GUIDE.md` if English UI copy is required. Operators can overlay machine or human translations, but the canonical database stores the source text. +- UI localisation: enable the translation bundles documented in `docs/UI_GUIDE.md` if English UI copy is required. Operators can overlay machine or human translations, but the canonical database stores the source text. - Docs guild is compiling a CERT-Bund terminology glossary under `docs/locale/certbund-glossary.md` so downstream teams can reference consistent English equivalents without altering the stored advisories. --- diff --git a/docs/modules/cryptography/README.md b/docs/modules/cryptography/README.md index d2a2fe0a6..d37082676 100644 --- a/docs/modules/cryptography/README.md +++ b/docs/modules/cryptography/README.md @@ -42,7 +42,7 @@ Key features: - Signer Module: `../signer/` - Attestor Module: `../attestor/` - Authority Module: `../authority/` -- Air-Gap Operations: `../../24_OFFLINE_KIT.md` +- Air-Gap Operations: `../../OFFLINE_KIT.md` ## Current Status diff --git a/docs/modules/cryptography/architecture.md b/docs/modules/cryptography/architecture.md index 7e2e941a8..2421e23b4 100644 --- a/docs/modules/cryptography/architecture.md +++ b/docs/modules/cryptography/architecture.md @@ -295,4 +295,4 @@ For air-gapped deployments: * Multi-profile signing: `./multi-profile-signing-specification.md` * Signer module: `../signer/architecture.md` * Attestor module: `../attestor/architecture.md` -* Offline operations: `../../24_OFFLINE_KIT.md` +* Offline operations: `../../OFFLINE_KIT.md` diff --git a/docs/modules/evidence-locker/README.md b/docs/modules/evidence-locker/README.md index c134f0843..7311e4bca 100644 --- a/docs/modules/evidence-locker/README.md +++ b/docs/modules/evidence-locker/README.md @@ -41,7 +41,7 @@ Key settings: - Operations: `./operations/` (if exists) - ExportCenter: `../export-center/` - Attestor: `../attestor/` -- High-Level Architecture: `../../07_HIGH_LEVEL_ARCHITECTURE.md` +- High-Level Architecture: `../../ARCHITECTURE_OVERVIEW.md` ## Current Status diff --git a/docs/modules/excititor/mirrors.md b/docs/modules/excititor/mirrors.md index fbe77f686..9697218de 100644 --- a/docs/modules/excititor/mirrors.md +++ b/docs/modules/excititor/mirrors.md @@ -152,7 +152,7 @@ Downstream automation reads `manifest.json`/`bundle.json` directly, while `/exci * Track quota utilisation via HTTP 429 metrics (configure structured logging or OTEL counters when rate limiting triggers). * Mirror domains can be deployed per tenant (e.g., `tenant-a`, `tenant-b`) with different auth requirements. * Ensure the underlying artifact stores (`FileSystem`, `S3`, offline bundle) retain artefacts long enough for mirrors to sync. -* For air-gapped mirrors, combine mirror endpoints with the Offline Kit (see `docs/24_OFFLINE_KIT.md`). +* For air-gapped mirrors, combine mirror endpoints with the Offline Kit (see `docs/OFFLINE_KIT.md`). --- diff --git a/docs/modules/excititor/trust-lattice.md b/docs/modules/excititor/trust-lattice.md index 23e9d9adf..f348864f5 100644 --- a/docs/modules/excititor/trust-lattice.md +++ b/docs/modules/excititor/trust-lattice.md @@ -409,7 +409,7 @@ gates: | POST | `/api/v1/authority/verdicts/{manifestId}/replay` | Verify replay | | GET | `/api/v1/authority/verdicts/{manifestId}/download` | Download signed manifest | -See `docs/09_API_CLI_REFERENCE.md` for complete API documentation. +See `docs/API_CLI_REFERENCE.md` for complete API documentation. --- @@ -506,7 +506,7 @@ Note: Conflict recorded in audit trail - [Excititor Architecture](./architecture.md) - [Verdict Manifest Specification](../authority/verdict-manifest.md) - [Policy Gates Configuration](../policy/architecture.md) -- [API Reference](../../09_API_CLI_REFERENCE.md) +- [API Reference](../../API_CLI_REFERENCE.md) --- diff --git a/docs/modules/export-center/operations/runbook.md b/docs/modules/export-center/operations/runbook.md index 2c504d0a1..1e17a9550 100644 --- a/docs/modules/export-center/operations/runbook.md +++ b/docs/modules/export-center/operations/runbook.md @@ -200,6 +200,6 @@ If encryption enabled, decrypt using age or AES key before verification. - `docs/modules/export-center/mirror-bundles.md` - `ops/devops/TASKS.md` (`DEVOPS-EXPORT-36-001`, `DEVOPS-EXPORT-37-001`) - `docs/aoc/aggregation-only-contract.md` -- `docs/24_OFFLINE_KIT.md` +- `docs/OFFLINE_KIT.md` > **Imposed rule:** Work of this type or tasks of this type on this component must also be applied everywhere else it should be applied. diff --git a/docs/modules/export-center/overview.md b/docs/modules/export-center/overview.md index c9fbb5256..effccd2da 100644 --- a/docs/modules/export-center/overview.md +++ b/docs/modules/export-center/overview.md @@ -40,7 +40,7 @@ See `docs/security/policy-governance.md` and `docs/aoc/aggregation-only-contract - **Mirror bundles.** `mirror:full` packages raw evidence, normalized indexes, policy snapshots, and provenance in a portable filesystem layout suitable for disconnected environments. `mirror:delta` tracks changes relative to a prior export manifest. - **No unsanctioned egress.** The exporter respects the platform allowlist. External calls (e.g., OCI pushes) require explicit configuration and are disabled by default for offline installs. -Consult `docs/24_OFFLINE_KIT.md` for Offline Kit delivery and `docs/modules/concelier/operations/mirror.md` for mirror ingestion procedures. +Consult `docs/OFFLINE_KIT.md` for Offline Kit delivery and `docs/modules/concelier/operations/mirror.md` for mirror ingestion procedures. ## Getting started 1. **Choose a profile.** Map requirements to the profile table above. Policy-aware exports need a published policy snapshot. diff --git a/docs/modules/facet/architecture.md b/docs/modules/facet/architecture.md new file mode 100644 index 000000000..e4be1768f --- /dev/null +++ b/docs/modules/facet/architecture.md @@ -0,0 +1,700 @@ +# Facet Sealing Architecture + +> **Ownership:** Scanner Guild, Policy Guild +> **Audience:** Service owners, platform engineers, security architects +> **Related:** [Platform Architecture](../platform/architecture-overview.md), [Scanner Architecture](../scanner/architecture.md), [Replay Architecture](../replay/architecture.md), [Policy Engine](../policy/architecture.md) + +This dossier describes the Facet Sealing subsystem, which provides cryptographically sealed manifests for logical slices of container images, enabling fine-grained drift detection, per-facet quota enforcement, and deterministic change tracking. + +--- + +## 1. Overview + +A **Facet** is a declared logical slice of a container image representing a cohesive set of files with shared characteristics: + +| Facet Type | Description | Examples | +|------------|-------------|----------| +| `os` | Operating system packages | `/var/lib/dpkg/**`, `/var/lib/rpm/**` | +| `lang/` | Language-specific dependencies | `node_modules/**`, `site-packages/**`, `vendor/**` | +| `binary` | Native binaries and shared libraries | `/usr/bin/*`, `/lib/**/*.so*` | +| `config` | Configuration files | `/etc/**`, `*.conf`, `*.yaml` | +| `custom` | User-defined patterns | Project-specific paths | + +Each facet can be individually **sealed** (cryptographic snapshot) and monitored for **drift** (changes between seals). + +--- + +## 2. System Landscape + +```mermaid +graph TD + subgraph Scanner["Scanner Services"] + FE[FacetExtractor] + FH[FacetHasher] + MB[MerkleBuilder] + end + + subgraph Storage["Facet Storage"] + FS[(PostgreSQL
facet_seals)] + FC[(CAS
facet_manifests)] + end + + subgraph Policy["Policy & Enforcement"] + DC[DriftCalculator] + QE[QuotaEnforcer] + AV[AdmissionValidator] + end + + subgraph Signing["Attestation"] + DS[DSSE Signer] + AT[Attestor] + end + + subgraph CLI["CLI & Integration"] + SealCmd[stella seal] + DriftCmd[stella drift] + VexCmd[stella vex gen] + Zastava[Zastava Webhook] + end + + FE --> FH + FH --> MB + MB --> DS + DS --> FS + DS --> FC + FS --> DC + DC --> QE + QE --> AV + AV --> Zastava + SealCmd --> FE + DriftCmd --> DC + VexCmd --> DC +``` + +--- + +## 3. Core Data Models + +### 3.1 FacetDefinition + +Declares a facet with its extraction patterns and quota constraints: + +```csharp +public sealed record FacetDefinition +{ + public required string FacetId { get; init; } // e.g., "os", "lang/node", "binary" + public required FacetType Type { get; init; } // OS, LangNode, LangPython, Binary, Config, Custom + public required ImmutableArray IncludeGlobs { get; init; } + public ImmutableArray ExcludeGlobs { get; init; } = []; + public FacetQuota? Quota { get; init; } +} + +public enum FacetType +{ + OS, + LangNode, + LangPython, + LangGo, + LangRust, + LangJava, + LangDotNet, + Binary, + Config, + Custom +} +``` + +### 3.2 FacetManifest + +Per-facet file manifest with Merkle root: + +```csharp +public sealed record FacetManifest +{ + public required string FacetId { get; init; } + public required FacetType Type { get; init; } + public required ImmutableArray Files { get; init; } + public required string MerkleRoot { get; init; } // SHA-256 hex + public required int FileCount { get; init; } + public required long TotalBytes { get; init; } + public required DateTimeOffset ExtractedAt { get; init; } + public required string ExtractorVersion { get; init; } +} + +public sealed record FacetFileEntry +{ + public required string Path { get; init; } // Normalized POSIX path + public required string ContentHash { get; init; } // SHA-256 hex + public required long Size { get; init; } + public required string Mode { get; init; } // POSIX mode string "0644" + public required DateTimeOffset ModTime { get; init; } // Normalized to UTC +} +``` + +### 3.3 FacetSeal + +DSSE-signed seal combining manifest with metadata: + +```csharp +public sealed record FacetSeal +{ + public required Guid SealId { get; init; } + public required string ImageRef { get; init; } // registry/repo:tag@sha256:... + public required string ImageDigest { get; init; } // sha256:... + public required FacetManifest Manifest { get; init; } + public required DateTimeOffset SealedAt { get; init; } + public required string SealedBy { get; init; } // Identity/service + public required FacetQuota? AppliedQuota { get; init; } + public required DsseEnvelope Envelope { get; init; } +} +``` + +### 3.4 FacetQuota + +Per-facet change budget: + +```csharp +public sealed record FacetQuota +{ + public required string FacetId { get; init; } + public double MaxChurnPercent { get; init; } = 5.0; // 0-100 + public int MaxChangedFiles { get; init; } = 50; + public int MaxAddedFiles { get; init; } = 25; + public int MaxRemovedFiles { get; init; } = 10; + public QuotaAction OnExceed { get; init; } = QuotaAction.Warn; +} + +public enum QuotaAction +{ + Warn, // Log warning, allow admission + Block, // Reject admission + RequireVex // Require VEX justification before admission +} +``` + +### 3.5 FacetDrift + +Drift calculation result between two seals: + +```csharp +public sealed record FacetDrift +{ + public required string FacetId { get; init; } + public required Guid BaselineSealId { get; init; } + public required Guid CurrentSealId { get; init; } + public required ImmutableArray Added { get; init; } + public required ImmutableArray Removed { get; init; } + public required ImmutableArray Modified { get; init; } + public required DriftScore Score { get; init; } + public required QuotaVerdict QuotaVerdict { get; init; } +} + +public sealed record DriftEntry +{ + public required string Path { get; init; } + public string? OldHash { get; init; } + public string? NewHash { get; init; } + public long? OldSize { get; init; } + public long? NewSize { get; init; } + public DriftCause Cause { get; init; } = DriftCause.Unknown; +} + +public enum DriftCause +{ + Unknown, + PackageUpdate, + ConfigChange, + BinaryRebuild, + NewDependency, + RemovedDependency, + SecurityPatch +} + +public sealed record DriftScore +{ + public required int TotalChanges { get; init; } + public required double ChurnPercent { get; init; } + public required int AddedCount { get; init; } + public required int RemovedCount { get; init; } + public required int ModifiedCount { get; init; } +} + +public sealed record QuotaVerdict +{ + public required bool Passed { get; init; } + public required ImmutableArray Violations { get; init; } + public required QuotaAction RecommendedAction { get; init; } +} + +public sealed record QuotaViolation +{ + public required string QuotaField { get; init; } // e.g., "MaxChurnPercent" + public required double Limit { get; init; } + public required double Actual { get; init; } + public required string Message { get; init; } +} +``` + +--- + +## 4. Component Architecture + +### 4.1 FacetExtractor + +Extracts file entries from container images based on facet definitions: + +```csharp +public interface IFacetExtractor +{ + Task ExtractAsync( + string imageRef, + FacetDefinition definition, + CancellationToken ct = default); + + Task> ExtractAllAsync( + string imageRef, + ImmutableArray definitions, + CancellationToken ct = default); +} +``` + +Implementation notes: +- Uses existing `ISurfaceReader` for container layer traversal +- Normalizes paths to POSIX format (forward slashes, no trailing slashes) +- Computes SHA-256 content hashes for each file +- Normalizes timestamps to UTC, mode to POSIX string +- Sorts files lexicographically for deterministic ordering + +### 4.2 FacetHasher + +Computes Merkle tree for facet file entries: + +```csharp +public interface IFacetHasher +{ + FacetMerkleResult ComputeMerkle(ImmutableArray files); +} + +public sealed record FacetMerkleResult +{ + public required string Root { get; init; } + public required ImmutableArray LeafHashes { get; init; } + public required ImmutableArray Proof { get; init; } +} +``` + +Implementation notes: +- Leaf hash = SHA-256(path || contentHash || size || mode) +- Binary Merkle tree with lexicographic leaf ordering +- Empty facet produces well-known empty root hash +- Proof enables verification of individual file membership + +### 4.3 FacetSealStore + +PostgreSQL storage for sealed facet manifests: + +```sql +-- Core seal storage +CREATE TABLE facet_seals ( + seal_id UUID PRIMARY KEY, + tenant TEXT NOT NULL, + image_ref TEXT NOT NULL, + image_digest TEXT NOT NULL, + facet_id TEXT NOT NULL, + facet_type TEXT NOT NULL, + merkle_root TEXT NOT NULL, + file_count INTEGER NOT NULL, + total_bytes BIGINT NOT NULL, + sealed_at TIMESTAMPTZ NOT NULL, + sealed_by TEXT NOT NULL, + quota_json JSONB, + manifest_cas TEXT NOT NULL, -- CAS URI to full manifest + dsse_envelope JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uq_facet_seal UNIQUE (tenant, image_digest, facet_id) +); + +CREATE INDEX ix_facet_seals_image ON facet_seals (tenant, image_digest); +CREATE INDEX ix_facet_seals_merkle ON facet_seals (merkle_root); + +-- Drift history +CREATE TABLE facet_drift_history ( + drift_id UUID PRIMARY KEY, + tenant TEXT NOT NULL, + baseline_seal_id UUID NOT NULL REFERENCES facet_seals(seal_id), + current_seal_id UUID NOT NULL REFERENCES facet_seals(seal_id), + facet_id TEXT NOT NULL, + drift_score_json JSONB NOT NULL, + quota_verdict_json JSONB NOT NULL, + computed_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT uq_drift_pair UNIQUE (baseline_seal_id, current_seal_id) +); +``` + +### 4.4 DriftCalculator + +Computes drift between baseline and current seals: + +```csharp +public interface IDriftCalculator +{ + Task CalculateAsync( + Guid baselineSealId, + Guid currentSealId, + CancellationToken ct = default); + + Task> CalculateAllAsync( + string imageDigestBaseline, + string imageDigestCurrent, + CancellationToken ct = default); +} +``` + +Implementation notes: +- Retrieves manifests from CAS via seal metadata +- Performs set difference operations on file paths +- Detects modifications via content hash comparison +- Attributes drift causes where determinable (e.g., package manager metadata) + +### 4.5 QuotaEnforcer + +Evaluates drift against quota constraints: + +```csharp +public interface IQuotaEnforcer +{ + QuotaVerdict Evaluate(FacetDrift drift, FacetQuota quota); + + Task> EvaluateAllAsync( + ImmutableArray drifts, + ImmutableDictionary quotas, + CancellationToken ct = default); +} +``` + +### 4.6 AdmissionValidator + +Zastava webhook integration for admission control: + +```csharp +public interface IFacetAdmissionValidator +{ + Task ValidateAsync( + AdmissionRequest request, + CancellationToken ct = default); +} + +public sealed record AdmissionResult +{ + public required bool Allowed { get; init; } + public string? Message { get; init; } + public ImmutableArray Violations { get; init; } = []; + public string? RequiredVexStatement { get; init; } +} +``` + +--- + +## 5. DSSE Envelope Structure + +Facet seals use DSSE (Dead Simple Signing Envelope) for cryptographic binding: + +```json +{ + "payloadType": "application/vnd.stellaops.facet-seal.v1+json", + "payload": "", + "signatures": [ + { + "keyid": "sha256:abc123...", + "sig": "" + } + ] +} +``` + +Payload structure (canonical JSON, RFC 8785): +```json +{ + "_type": "https://stellaops.io/FacetSeal/v1", + "facetId": "os", + "facetType": "OS", + "imageDigest": "sha256:abc123...", + "imageRef": "registry.example.com/app:v1.2.3", + "manifest": { + "extractedAt": "2026-01-05T10:00:00.000Z", + "extractorVersion": "1.0.0", + "fileCount": 1234, + "files": [ + { + "contentHash": "sha256:...", + "mode": "0644", + "modTime": "2026-01-01T00:00:00.000Z", + "path": "/etc/os-release", + "size": 256 + } + ], + "merkleRoot": "sha256:def456...", + "totalBytes": 1048576 + }, + "quota": { + "maxAddedFiles": 25, + "maxChangedFiles": 50, + "maxChurnPercent": 5.0, + "maxRemovedFiles": 10, + "onExceed": "Warn" + }, + "sealId": "550e8400-e29b-41d4-a716-446655440000", + "sealedAt": "2026-01-05T10:05:00.000Z", + "sealedBy": "scanner-worker-01" +} +``` + +--- + +## 6. Default Facet Definitions + +Standard facet definitions applied when no custom configuration is provided: + +```yaml +# Default facet configuration +facets: + - facetId: os + type: OS + includeGlobs: + - /var/lib/dpkg/** + - /var/lib/rpm/** + - /var/lib/pacman/** + - /var/lib/apk/** + - /var/cache/apt/** + - /etc/apt/** + - /etc/yum.repos.d/** + excludeGlobs: + - "**/*.log" + quota: + maxChurnPercent: 5.0 + maxChangedFiles: 100 + onExceed: Warn + + - facetId: lang/node + type: LangNode + includeGlobs: + - "**/node_modules/**" + - "**/package.json" + - "**/package-lock.json" + - "**/yarn.lock" + - "**/pnpm-lock.yaml" + quota: + maxChurnPercent: 10.0 + maxChangedFiles: 500 + onExceed: RequireVex + + - facetId: lang/python + type: LangPython + includeGlobs: + - "**/site-packages/**" + - "**/dist-packages/**" + - "**/requirements.txt" + - "**/Pipfile.lock" + - "**/poetry.lock" + quota: + maxChurnPercent: 10.0 + maxChangedFiles: 200 + onExceed: Warn + + - facetId: lang/go + type: LangGo + includeGlobs: + - "**/go.mod" + - "**/go.sum" + - "**/vendor/**" + quota: + maxChurnPercent: 15.0 + maxChangedFiles: 100 + onExceed: Warn + + - facetId: binary + type: Binary + includeGlobs: + - /usr/bin/* + - /usr/sbin/* + - /bin/* + - /sbin/* + - /usr/lib/**/*.so* + - /lib/**/*.so* + - /usr/local/bin/* + excludeGlobs: + - "**/*.py" + - "**/*.sh" + quota: + maxChurnPercent: 2.0 + maxChangedFiles: 20 + onExceed: Block + + - facetId: config + type: Config + includeGlobs: + - /etc/** + - "**/*.conf" + - "**/*.cfg" + - "**/*.ini" + - "**/*.yaml" + - "**/*.yml" + - "**/*.json" + excludeGlobs: + - /etc/passwd + - /etc/shadow + - /etc/group + - "**/*.log" + quota: + maxChurnPercent: 20.0 + maxChangedFiles: 50 + onExceed: Warn +``` + +--- + +## 7. Integration Points + +### 7.1 Scanner Integration + +Scanner invokes facet extraction during scan: + +```csharp +// In ScanOrchestrator +var facetDefs = await _facetConfigLoader.LoadAsync(scanRequest.FacetConfig, ct); +var manifests = await _facetExtractor.ExtractAllAsync(imageRef, facetDefs, ct); + +foreach (var manifest in manifests) +{ + var seal = await _facetSealer.SealAsync(manifest, scanRequest, ct); + await _facetSealStore.SaveAsync(seal, ct); +} +``` + +### 7.2 CLI Integration + +```bash +# Seal all facets for an image +stella seal myregistry.io/app:v1.2.3 --output seals.json + +# Seal specific facets +stella seal myregistry.io/app:v1.2.3 --facet os --facet lang/node + +# Check drift between two image versions +stella drift myregistry.io/app:v1.2.3 myregistry.io/app:v1.2.4 --format json + +# Generate VEX from drift +stella vex gen --from-drift myregistry.io/app:v1.2.3 myregistry.io/app:v1.2.4 +``` + +### 7.3 Zastava Webhook Integration + +```csharp +// In FacetAdmissionValidator +public async Task ValidateAsync(AdmissionRequest request, CancellationToken ct) +{ + // Find baseline seal (latest approved) + var baseline = await _sealStore.GetLatestApprovedAsync(request.ImageRef, ct); + if (baseline is null) + return AdmissionResult.Allowed("No baseline seal found, skipping facet check"); + + // Extract current facets + var currentManifests = await _extractor.ExtractAllAsync(request.ImageRef, _defaultFacets, ct); + + // Calculate drift for each facet + var drifts = new List(); + foreach (var manifest in currentManifests) + { + var baselineSeal = baseline.FirstOrDefault(s => s.FacetId == manifest.FacetId); + if (baselineSeal is not null) + { + var drift = await _driftCalculator.CalculateAsync(baselineSeal, manifest, ct); + drifts.Add(drift); + } + } + + // Evaluate quotas + var violations = new List(); + QuotaAction maxAction = QuotaAction.Warn; + + foreach (var drift in drifts) + { + var verdict = _quotaEnforcer.Evaluate(drift, drift.AppliedQuota); + if (!verdict.Passed) + { + violations.AddRange(verdict.Violations); + if (verdict.RecommendedAction > maxAction) + maxAction = verdict.RecommendedAction; + } + } + + return maxAction switch + { + QuotaAction.Block => AdmissionResult.Denied(violations), + QuotaAction.RequireVex => AdmissionResult.RequiresVex(violations), + _ => AdmissionResult.Allowed(violations) + }; +} +``` + +--- + +## 8. Observability + +### 8.1 Metrics + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `facet_seal_total` | Counter | `tenant`, `facet_type`, `status` | Total seals created | +| `facet_seal_duration_seconds` | Histogram | `facet_type` | Time to create seal | +| `facet_drift_score` | Gauge | `tenant`, `facet_id`, `image` | Current drift score | +| `facet_quota_violations_total` | Counter | `tenant`, `facet_id`, `quota_field` | Quota violations | +| `facet_admission_decisions_total` | Counter | `tenant`, `decision`, `facet_id` | Admission decisions | + +### 8.2 Traces + +``` +facet.extract - Facet file extraction from image +facet.hash - Merkle tree computation +facet.seal - DSSE signing +facet.drift.compute - Drift calculation +facet.quota.evaluate - Quota enforcement +facet.admission - Admission validation +``` + +### 8.3 Logs + +Structured log fields: +- `facetId`: Facet identifier +- `imageRef`: Container image reference +- `imageDigest`: Image content digest +- `merkleRoot`: Facet Merkle root +- `driftScore`: Computed drift percentage +- `quotaVerdict`: Pass/fail status + +--- + +## 9. Security Considerations + +1. **Signature Verification**: All seals must be DSSE-signed with keys managed by Authority service +2. **Tenant Isolation**: Seals are scoped to tenants; cross-tenant access is prohibited +3. **Immutability**: Once created, seals cannot be modified; only superseded by new seals +4. **Audit Trail**: All seal operations are logged with correlation IDs +5. **Key Rotation**: Signing keys support rotation; old signatures remain valid with archived keys + +--- + +## 10. References + +- [DSSE Specification](https://github.com/secure-systems-lab/dsse) +- [RFC 8785 - JSON Canonicalization](https://tools.ietf.org/html/rfc8785) +- [Scanner Architecture](../scanner/architecture.md) +- [Attestor Architecture](../attestor/architecture.md) +- [Policy Engine Architecture](../policy/architecture.md) +- [Replay Architecture](../replay/architecture.md) + +--- + +*Last updated: 2026-01-05* diff --git a/docs/modules/findings-ledger/operations/rls-migration.md b/docs/modules/findings-ledger/operations/rls-migration.md index b83b2c4a7..ee7f754ae 100644 --- a/docs/modules/findings-ledger/operations/rls-migration.md +++ b/docs/modules/findings-ledger/operations/rls-migration.md @@ -165,7 +165,7 @@ jobs: - [Tenant Isolation & Redaction](../tenant-isolation-redaction.md) - [Findings Ledger Deployment](../deployment.md) -- [Offline Kit Operations](../../../24_OFFLINE_KIT.md) +- [Offline Kit Operations](../../../OFFLINE_KIT.md) --- diff --git a/docs/modules/gateway/README.md b/docs/modules/gateway/README.md index 04e065431..01df37e22 100644 --- a/docs/modules/gateway/README.md +++ b/docs/modules/gateway/README.md @@ -42,7 +42,7 @@ Key settings: - Architecture: `./architecture.md` - Router Module: `../router/` - Authority Module: `../authority/` -- API Reference: `../../09_API_CLI_REFERENCE.md` +- API Reference: `../../API_CLI_REFERENCE.md` ## Current Status diff --git a/docs/modules/mirror/README.md b/docs/modules/mirror/README.md index 7b3140c92..ef46924c1 100644 --- a/docs/modules/mirror/README.md +++ b/docs/modules/mirror/README.md @@ -38,7 +38,7 @@ Key features: - AirGap Module: `../airgap/` - ExportCenter: `../export-center/` -- Offline Kit: `../../24_OFFLINE_KIT.md` +- Offline Kit: `../../OFFLINE_KIT.md` - Operations: `./operations/` (if exists) ## Current Status diff --git a/docs/modules/packsregistry/architecture.md b/docs/modules/packs-registry/architecture.md similarity index 100% rename from docs/modules/packsregistry/architecture.md rename to docs/modules/packs-registry/architecture.md diff --git a/docs/modules/platform/architecture-overview.md b/docs/modules/platform/architecture-overview.md index e69dc8c09..009826d57 100644 --- a/docs/modules/platform/architecture-overview.md +++ b/docs/modules/platform/architecture-overview.md @@ -1,7 +1,7 @@ -# StellaOps Architecture Overview (Sprint 19) +# StellaOps Architecture Overview -> **Ownership:** Architecture Guild • Docs Guild -> **Audience:** Service owners, platform engineers, solution architects +> **Ownership:** Architecture Guild • Docs Guild +> **Audience:** Service owners, platform engineers, solution architects > **Related:** [High-Level Architecture](../../ARCHITECTURE_REFERENCE.md), [Concelier Architecture](../concelier/architecture.md), [Policy Engine Architecture](../policy/architecture.md), [Aggregation-Only Contract](../../aoc/aggregation-only-contract.md) This dossier summarises the end-to-end runtime topology after the Aggregation-Only Contract (AOC) rollout. It highlights where raw facts live, how ingest services enforce guardrails, and how downstream components consume those facts to derive policy decisions and user-facing experiences. @@ -27,7 +27,7 @@ This dossier summarises the end-to-end runtime topology after the Aggregation-On > Evaluate public scanner incidents? The [Ecosystem Test Cases](../product-advisories/30-Nov-2025 - Ecosystem Test Cases for StellaOps.md) document five hardened regressions (Grype credential leak, Trivy offline schema, SBOM parity, Grype instability) that you can turn into acceptance tests today. -## 1 · System landscape +## 1 · System landscape ```mermaid graph TD @@ -94,7 +94,7 @@ Key boundaries: --- -## 2 · Aggregation-Only Contract focus +## 2 · Aggregation-Only Contract focus ### 2.1 Responsibilities at the boundary @@ -146,7 +146,7 @@ sequenceDiagram --- -## 3 · Data & control flow highlights +## 3 · Data & control flow highlights 1. **Ingestion:** Concelier / Excititor connectors fetch upstream documents, compute linksets, and hand payloads to `AOCWriteGuard`. Guards validate schema, provenance, forbidden fields, supersedes pointers, and append-only rules before writing to PostgreSQL. 2. **Verification:** `stella aoc verify` (CLI/CI) and `/aoc/verify` endpoints replay guard checks against stored documents, mapping `ERR_AOC_00x` codes to exit codes for automation. @@ -156,7 +156,7 @@ sequenceDiagram --- -## 4 · Offline & disaster readiness +## 4 · Offline & disaster readiness - **Offline Kit:** Packages raw PostgreSQL snapshots (`advisory_raw`, `vex_raw`) plus guard configuration and CLI verifier binaries so air-gapped sites can re-run AOC checks before promotion. - **Recovery:** Supersedes chains allow rollback to prior revisions without mutating rows. Disaster exercises must rehearse restoring from snapshot, replaying logical replication into Policy Engine, and re-validating guard compliance. @@ -164,9 +164,9 @@ sequenceDiagram --- -## 5 · Replay CAS & deterministic bundles +## 5 · Replay CAS & deterministic bundles -- **Replay CAS:** Content-addressed storage lives under `cas://replay//.tar.zst`. Writers must use [StellaOps.Replay.Core](../../src/__Libraries/StellaOps.Replay.Core/AGENTS.md) helpers to ensure lexicographic file ordering, POSIX mode normalisation (0644/0755), LF newlines, zstd level 19 compression, and shard-by-prefix CAS URIs (`BuildCasUri`). Bundle metadata (size, hash, created) feeds the platform-wide `replay_bundles` collection defined in `docs/db/replay-schema.md`. +- **Replay CAS:** Content-addressed storage lives under `cas://replay//.tar.zst`. Writers must use [StellaOps.Replay.Core](../../src/__Libraries/StellaOps.Replay.Core/AGENTS.md) helpers to ensure lexicographic file ordering, POSIX mode normalisation (0644/0755), LF newlines, zstd level 19 compression, and shard-by-prefix CAS URIs (`BuildCasUri`). Bundle metadata (size, hash, created) feeds the platform-wide `replay_bundles` collection defined in `docs/db/replay-schema.md`. - **Artifacts:** Each recorded scan stores three bundles: 1. `manifest.json` (canonical JSON, hashed and signed via DSSE). 2. `inputbundle.tar.zst` (feeds, policies, tools, environment snapshot). @@ -175,11 +175,11 @@ sequenceDiagram - **Reachability subtree:** When reachability recording is enabled, Scanner uploads graphs & runtime traces under `cas://replay//reachability/graphs/` and `cas://replay//reachability/traces/`. Manifest references (StellaOps.Replay.Core) bind these URIs along with analyzer hashes so Replay + Signals can rehydrate explainability evidence deterministically. - **Storage tiers:** Primary storage is PostgreSQL (`replay_runs`, `replay_subjects`) plus the CAS bucket. Evidence Locker mirrors bundles for long-term retention and legal hold workflows (`docs/modules/evidence-locker/architecture.md`). Offline kits package bundles under `offline/replay/` with detached DSSE envelopes for air-gapped verification. - **APIs & ownership:** Scanner WebService produces the bundles via `record` mode, Scanner Worker emits Merkle metadata, Signer/Authority provide DSSE signatures, Attestor anchors manifests to Rekor, CLI/Evidence Locker handle retrieval, and Docs Guild maintains runbooks. Responsibilities are tracked in `docs/implplan/SPRINT_185_shared_replay_primitives.md` through `SPRINT_187_evidence_locker_cli_integration.md`. -- **Operational policies:** Retention defaults to 180 days for hot CAS storage and 2 years for cold Evidence Locker copies. Rotation and pruning follow the checklist in `docs/runbooks/replay_ops.md`. +- **Operational policies:** Retention defaults to 180 days for hot CAS storage and 2 years for cold Evidence Locker copies. Rotation and pruning follow the checklist in `docs/runbooks/replay_ops.md`. --- -## 6 · References +## 6 · References - [Aggregation-Only Contract reference](../../aoc/aggregation-only-contract.md) - [Concelier architecture](../concelier/architecture.md) @@ -194,7 +194,7 @@ sequenceDiagram --- -## 7 · Compliance checklist +## 7 · Compliance checklist - [ ] AOC guard enabled for all Concelier and Excititor write paths in production. - [ ] PostgreSQL schema constraints deployed for `advisory_raw` and `vex_raw`; logical replication scoped per tenant. @@ -208,4 +208,4 @@ sequenceDiagram --- -*Last updated: 2025-12-23 (Testing strategy links and catalog).* +*Last updated: 2026-01-05 (Removed dated sprint reference).* diff --git a/docs/modules/provcache/README.md b/docs/modules/prov-cache/README.md similarity index 99% rename from docs/modules/provcache/README.md rename to docs/modules/prov-cache/README.md index fc372b636..cd71301bc 100644 --- a/docs/modules/provcache/README.md +++ b/docs/modules/prov-cache/README.md @@ -617,6 +617,6 @@ CREATE INDEX idx_revocations_time ON provcache.prov_revocations(revoked_at); - **[Provcache Architecture Guide](architecture.md)** - Detailed architecture, invalidation flows, and API reference - [Policy Engine Architecture](../policy/README.md) - [TrustLattice Engine](../policy/design/policy-deterministic-evaluator.md) -- [Offline Kit Documentation](../../24_OFFLINE_KIT.md) +- [Offline Kit Documentation](../../OFFLINE_KIT.md) - [Air-Gap Controller](../airgap/README.md) - [Authority Key Rotation](../authority/README.md) diff --git a/docs/modules/provcache/architecture.md b/docs/modules/prov-cache/architecture.md similarity index 100% rename from docs/modules/provcache/architecture.md rename to docs/modules/prov-cache/architecture.md diff --git a/docs/modules/provcache/metrics-alerting.md b/docs/modules/prov-cache/metrics-alerting.md similarity index 100% rename from docs/modules/provcache/metrics-alerting.md rename to docs/modules/prov-cache/metrics-alerting.md diff --git a/docs/modules/provcache/oci-attestation-verification.md b/docs/modules/prov-cache/oci-attestation-verification.md similarity index 100% rename from docs/modules/provcache/oci-attestation-verification.md rename to docs/modules/prov-cache/oci-attestation-verification.md diff --git a/docs/modules/reachgraph/architecture.md b/docs/modules/reach-graph/architecture.md similarity index 100% rename from docs/modules/reachgraph/architecture.md rename to docs/modules/reach-graph/architecture.md diff --git a/docs/modules/replay/replay-proof-schema.md b/docs/modules/replay/replay-proof-schema.md new file mode 100644 index 000000000..42aeddbc3 --- /dev/null +++ b/docs/modules/replay/replay-proof-schema.md @@ -0,0 +1,606 @@ +# Replay Proof Schema + +> **Ownership:** Replay Guild, Scanner Guild, Attestor Guild +> **Audience:** Service owners, platform engineers, auditors, compliance teams +> **Related:** [Platform Architecture](../platform/architecture-overview.md), [Replay Architecture](./architecture.md), [Facet Sealing](../facet/architecture.md), [DSSE Specification](https://github.com/secure-systems-lab/dsse) + +This document defines the schema for Replay Proofs - compact, cryptographically verifiable artifacts that attest to deterministic policy evaluation outcomes. + +--- + +## 1. Overview + +A **Replay Proof** is a DSSE-signed artifact that proves a policy evaluation produced a specific verdict given a specific set of inputs. Replay proofs enable: + +- **Audit trails**: Compact proof that a verdict was computed correctly +- **Determinism verification**: Re-running with same inputs produces identical output +- **Time-travel debugging**: Understand why a past decision was made +- **Compliance evidence**: Cryptographic proof for regulatory requirements + +--- + +## 2. Replay Bundle Structure + +A complete replay bundle consists of three artifacts stored in CAS: + +``` +cas://replay// + manifest.json # DSSE-signed manifest (this document's focus) + inputbundle.tar.zst # Compressed input artifacts + outputbundle.tar.zst # Compressed output artifacts +``` + +### 2.1 Directory Layout + +``` +/ + manifest.json + inputbundle.tar.zst + feeds/ + nvd/.json + osv/.json + ghsa/.json + policy/ + bundle.tar + version.json + sboms/ + .spdx.json + .cdx.json + vex/ + .openvex.json + config/ + lattice.json + feature-flags.json + seeds/ + random-seeds.json + clock-offsets.json + outputbundle.tar.zst + verdicts/ + .json + findings/ + .json + merkle/ + verdict-tree.json + finding-tree.json + logs/ + replay.log + trace.json +``` + +--- + +## 3. Core Schema Definitions + +### 3.1 ReplayProof + +The primary proof artifact - a compact summary suitable for verification: + +```csharp +public sealed record ReplayProof +{ + // Identity + public required Guid ProofId { get; init; } + public required Guid RunId { get; init; } + public required string Subject { get; init; } // Image digest or SBOM ID + + // Input digest + public required KnowledgeSnapshotDigest InputDigest { get; init; } + + // Output digest + public required VerdictDigest OutputDigest { get; init; } + + // Execution metadata + public required ExecutionMetadata Execution { get; init; } + + // CAS references + public required BundleReferences Bundles { get; init; } + + // Signature + public required DateTimeOffset SignedAt { get; init; } + public required string SignedBy { get; init; } +} +``` + +### 3.2 KnowledgeSnapshotDigest + +Cryptographic digest of all inputs: + +```csharp +public sealed record KnowledgeSnapshotDigest +{ + // Component digests + public required string SbomsDigest { get; init; } // SHA-256 of sorted SBOM hashes + public required string VexDigest { get; init; } // SHA-256 of sorted VEX hashes + public required string FeedsDigest { get; init; } // SHA-256 of feed version manifest + public required string PolicyDigest { get; init; } // SHA-256 of policy bundle + public required string LatticeDigest { get; init; } // SHA-256 of lattice config + public required string SeedsDigest { get; init; } // SHA-256 of random seeds + + // Combined root + public required string RootDigest { get; init; } // SHA-256 of all component digests + + // Counts for quick comparison + public required int SbomCount { get; init; } + public required int VexCount { get; init; } + public required int FeedCount { get; init; } +} +``` + +### 3.3 VerdictDigest + +Cryptographic digest of all outputs: + +```csharp +public sealed record VerdictDigest +{ + public required string VerdictMerkleRoot { get; init; } // Merkle root of verdicts + public required string FindingMerkleRoot { get; init; } // Merkle root of findings + public required int VerdictCount { get; init; } + public required int FindingCount { get; init; } + public required VerdictSummary Summary { get; init; } +} + +public sealed record VerdictSummary +{ + public required int Critical { get; init; } + public required int High { get; init; } + public required int Medium { get; init; } + public required int Low { get; init; } + public required int Informational { get; init; } + public required int Suppressed { get; init; } + public required int Total { get; init; } +} +``` + +### 3.4 ExecutionMetadata + +Execution environment and timing: + +```csharp +public sealed record ExecutionMetadata +{ + // Timing + public required DateTimeOffset StartedAt { get; init; } + public required DateTimeOffset CompletedAt { get; init; } + public required long DurationMs { get; init; } + + // Engine version + public required EngineVersion Engine { get; init; } + + // Environment + public required string HostId { get; init; } + public required string RuntimeVersion { get; init; } // e.g., ".NET 10.0.0" + public required string Platform { get; init; } // e.g., "linux-x64" + + // Determinism markers + public required bool DeterministicMode { get; init; } + public required string ClockMode { get; init; } // "frozen", "simulated", "real" + public required string RandomMode { get; init; } // "seeded", "recorded", "real" +} + +public sealed record EngineVersion +{ + public required string Name { get; init; } // e.g., "PolicyEngine" + public required string Version { get; init; } // e.g., "2.1.0" + public required string SourceDigest { get; init; } // SHA-256 of engine source/binary +} +``` + +### 3.5 BundleReferences + +CAS URIs to full bundles: + +```csharp +public sealed record BundleReferences +{ + public required string ManifestUri { get; init; } // cas://replay//manifest.json + public required string InputBundleUri { get; init; } // cas://replay//inputbundle.tar.zst + public required string OutputBundleUri { get; init; } // cas://replay//outputbundle.tar.zst + public required string ManifestDigest { get; init; } // SHA-256 of manifest.json + public required string InputBundleDigest { get; init; } // SHA-256 of inputbundle.tar.zst + public required string OutputBundleDigest { get; init; } // SHA-256 of outputbundle.tar.zst + public required long InputBundleSize { get; init; } + public required long OutputBundleSize { get; init; } +} +``` + +--- + +## 4. DSSE Envelope + +Replay proofs are wrapped in DSSE envelopes for cryptographic binding: + +```json +{ + "payloadType": "application/vnd.stellaops.replay-proof.v1+json", + "payload": "", + "signatures": [ + { + "keyid": "sha256:abc123...", + "sig": "" + } + ] +} +``` + +### 4.1 Payload Type URI + +- **v1**: `application/vnd.stellaops.replay-proof.v1+json` +- **in-toto compatible**: `https://stellaops.io/ReplayProof/v1` + +### 4.2 Canonical JSON Encoding + +Payloads MUST be encoded using RFC 8785 canonical JSON: + +1. Keys sorted lexicographically using Unicode code points +2. No whitespace between structural characters +3. No trailing commas +4. Numbers without unnecessary decimal points or exponents +5. Strings with minimal escaping (only required characters) + +--- + +## 5. Full Manifest Schema + +The `manifest.json` file contains the complete proof plus additional metadata: + +```json +{ + "_type": "https://stellaops.io/ReplayManifest/v1", + "proofId": "550e8400-e29b-41d4-a716-446655440000", + "runId": "660e8400-e29b-41d4-a716-446655440001", + "subject": "sha256:abc123def456...", + "tenant": "acme-corp", + "inputDigest": { + "sbomsDigest": "sha256:111...", + "vexDigest": "sha256:222...", + "feedsDigest": "sha256:333...", + "policyDigest": "sha256:444...", + "latticeDigest": "sha256:555...", + "seedsDigest": "sha256:666...", + "rootDigest": "sha256:aaa...", + "sbomCount": 1, + "vexCount": 5, + "feedCount": 3 + }, + "outputDigest": { + "verdictMerkleRoot": "sha256:bbb...", + "findingMerkleRoot": "sha256:ccc...", + "verdictCount": 42, + "findingCount": 156, + "summary": { + "critical": 2, + "high": 8, + "medium": 25, + "low": 12, + "informational": 3, + "suppressed": 106, + "total": 156 + } + }, + "execution": { + "startedAt": "2026-01-05T10:00:00.000Z", + "completedAt": "2026-01-05T10:00:05.123Z", + "durationMs": 5123, + "engine": { + "name": "PolicyEngine", + "version": "2.1.0", + "sourceDigest": "sha256:engine123..." + }, + "hostId": "scanner-worker-01", + "runtimeVersion": ".NET 10.0.0", + "platform": "linux-x64", + "deterministicMode": true, + "clockMode": "frozen", + "randomMode": "seeded" + }, + "bundles": { + "manifestUri": "cas://replay/660e8400.../manifest.json", + "inputBundleUri": "cas://replay/660e8400.../inputbundle.tar.zst", + "outputBundleUri": "cas://replay/660e8400.../outputbundle.tar.zst", + "manifestDigest": "sha256:manifest...", + "inputBundleDigest": "sha256:input...", + "outputBundleDigest": "sha256:output...", + "inputBundleSize": 10485760, + "outputBundleSize": 2097152 + }, + "signedAt": "2026-01-05T10:00:06.000Z", + "signedBy": "scanner-worker-01" +} +``` + +--- + +## 6. Verification Protocol + +### 6.1 Quick Verification (Proof Only) + +Verify the DSSE signature and check digest consistency: + +```csharp +public async Task VerifyProofAsync( + ReplayProof proof, + DsseEnvelope envelope, + CancellationToken ct) +{ + // 1. Verify DSSE signature + var sigValid = await _dsseVerifier.VerifyAsync(envelope, ct); + if (!sigValid) + return VerificationResult.Failed("DSSE signature invalid"); + + // 2. Verify input digest consistency + var inputRoot = ComputeInputRoot( + proof.InputDigest.SbomsDigest, + proof.InputDigest.VexDigest, + proof.InputDigest.FeedsDigest, + proof.InputDigest.PolicyDigest, + proof.InputDigest.LatticeDigest, + proof.InputDigest.SeedsDigest); + + if (inputRoot != proof.InputDigest.RootDigest) + return VerificationResult.Failed("Input root digest mismatch"); + + return VerificationResult.Passed(); +} +``` + +### 6.2 Full Verification (With Replay) + +Download bundles and re-execute to verify determinism: + +```csharp +public async Task VerifyWithReplayAsync( + ReplayProof proof, + CancellationToken ct) +{ + // 1. Quick verification first + var quickResult = await VerifyProofAsync(proof, envelope, ct); + if (!quickResult.Passed) + return quickResult; + + // 2. Download bundles from CAS + var inputBundle = await _cas.DownloadAsync(proof.Bundles.InputBundleUri, ct); + var outputBundle = await _cas.DownloadAsync(proof.Bundles.OutputBundleUri, ct); + + // 3. Verify bundle digests + if (ComputeDigest(inputBundle) != proof.Bundles.InputBundleDigest) + return VerificationResult.Failed("Input bundle digest mismatch"); + if (ComputeDigest(outputBundle) != proof.Bundles.OutputBundleDigest) + return VerificationResult.Failed("Output bundle digest mismatch"); + + // 4. Extract and verify individual input digests + var inputs = await ExtractInputsAsync(inputBundle, ct); + var computedInputDigest = ComputeKnowledgeDigest(inputs); + if (computedInputDigest.RootDigest != proof.InputDigest.RootDigest) + return VerificationResult.Failed("Computed input digest mismatch"); + + // 5. Re-execute policy evaluation + var replayResult = await _replayEngine.ExecuteAsync(inputs, ct); + + // 6. Compare output digests + var computedOutputDigest = ComputeVerdictDigest(replayResult); + if (computedOutputDigest.VerdictMerkleRoot != proof.OutputDigest.VerdictMerkleRoot) + return VerificationResult.Failed("Verdict Merkle root mismatch - non-deterministic!"); + + if (computedOutputDigest.FindingMerkleRoot != proof.OutputDigest.FindingMerkleRoot) + return VerificationResult.Failed("Finding Merkle root mismatch - non-deterministic!"); + + return VerificationResult.Passed(); +} +``` + +--- + +## 7. Digest Computation + +### 7.1 Input Root Digest + +```csharp +public string ComputeInputRoot( + string sbomsDigest, + string vexDigest, + string feedsDigest, + string policyDigest, + string latticeDigest, + string seedsDigest) +{ + // Concatenate in fixed order with separators + var combined = string.Join("|", + sbomsDigest, + vexDigest, + feedsDigest, + policyDigest, + latticeDigest, + seedsDigest); + + return ComputeSha256(combined); +} +``` + +### 7.2 SBOM Collection Digest + +```csharp +public string ComputeSbomsDigest(IEnumerable sboms) +{ + // Sort by ID for determinism + var sorted = sboms.OrderBy(s => s.SbomId, StringComparer.Ordinal); + + // Concatenate hashes + var combined = string.Join("|", sorted.Select(s => s.ContentHash)); + + return ComputeSha256(combined); +} +``` + +### 7.3 Verdict Merkle Root + +```csharp +public string ComputeVerdictMerkleRoot(IEnumerable verdicts) +{ + // Sort by verdict ID for determinism + var sorted = verdicts.OrderBy(v => v.VerdictId, StringComparer.Ordinal); + + // Compute leaf hashes + var leaves = sorted.Select(v => ComputeVerdictLeafHash(v)).ToArray(); + + // Build Merkle tree + return MerkleTreeBuilder.ComputeRoot(leaves); +} + +private string ComputeVerdictLeafHash(Verdict verdict) +{ + var canonical = CanonicalJsonSerializer.Serialize(verdict); + return ComputeSha256(canonical); +} +``` + +--- + +## 8. Database Schema + +```sql +-- Replay proof storage +CREATE TABLE replay_proofs ( + proof_id UUID PRIMARY KEY, + run_id UUID NOT NULL, + tenant TEXT NOT NULL, + subject TEXT NOT NULL, + input_root_digest TEXT NOT NULL, + output_verdict_root TEXT NOT NULL, + output_finding_root TEXT NOT NULL, + execution_json JSONB NOT NULL, + bundles_json JSONB NOT NULL, + dsse_envelope JSONB NOT NULL, + signed_at TIMESTAMPTZ NOT NULL, + signed_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uq_replay_run UNIQUE (run_id) +); + +CREATE INDEX ix_replay_proofs_tenant ON replay_proofs (tenant, created_at DESC); +CREATE INDEX ix_replay_proofs_subject ON replay_proofs (subject); +CREATE INDEX ix_replay_proofs_input ON replay_proofs (input_root_digest); + +-- Replay verification log +CREATE TABLE replay_verifications ( + verification_id UUID PRIMARY KEY, + proof_id UUID NOT NULL REFERENCES replay_proofs(proof_id), + tenant TEXT NOT NULL, + verification_type TEXT NOT NULL, -- 'quick', 'full' + passed BOOLEAN NOT NULL, + failure_reason TEXT, + duration_ms BIGINT NOT NULL, + verified_at TIMESTAMPTZ NOT NULL, + verified_by TEXT NOT NULL, + + CONSTRAINT fk_proof FOREIGN KEY (proof_id) REFERENCES replay_proofs(proof_id) +); + +CREATE INDEX ix_replay_verifications_proof ON replay_verifications (proof_id); +``` + +--- + +## 9. CLI Integration + +```bash +# Verify a replay proof (quick - signature only) +stella verify --proof proof.json + +# Verify with full replay +stella verify --proof proof.json --replay + +# Verify from CAS URI +stella verify --bundle cas://replay/660e8400.../manifest.json + +# Export proof for audit +stella replay export --run-id 660e8400-... --output proof.json + +# List proofs for an image +stella replay list --subject sha256:abc123... + +# Diff two replay results +stella replay diff --run-id-a 660e8400... --run-id-b 770e8400... +``` + +--- + +## 10. API Endpoints + +```http +# Get proof by run ID +GET /api/v1/replay/{runId}/proof +Response: ReplayProof (JSON) + +# Verify proof +POST /api/v1/replay/{runId}/verify +Request: { "type": "quick" | "full" } +Response: VerificationResult + +# List proofs for subject +GET /api/v1/replay/proofs?subject={digest}&tenant={tenant} +Response: ReplayProofSummary[] + +# Download bundle +GET /api/v1/replay/{runId}/bundles/{type} +Response: Binary stream (tar.zst) + +# Compare two runs +GET /api/v1/replay/diff?runIdA={id}&runIdB={id} +Response: ReplayDiffResult +``` + +--- + +## 11. Error Codes + +| Code | Description | +|------|-------------| +| `REPLAY_001` | Proof not found | +| `REPLAY_002` | DSSE signature verification failed | +| `REPLAY_003` | Input digest mismatch | +| `REPLAY_004` | Output digest mismatch (non-deterministic) | +| `REPLAY_005` | Bundle not found in CAS | +| `REPLAY_006` | Bundle digest mismatch | +| `REPLAY_007` | Engine version mismatch | +| `REPLAY_008` | Replay execution failed | +| `REPLAY_009` | Insufficient permissions | +| `REPLAY_010` | Bundle format invalid | + +--- + +## 12. Migration from v0 + +If upgrading from pre-v1 replay bundles: + +1. **Schema migration**: Run `migrate-replay-schema.sql` +2. **Re-sign existing proofs**: Use `stella replay migrate --sign` to add DSSE envelopes +3. **Verify migration**: Run `stella replay verify --all` to check integrity +4. **Update consumers**: Point to new `/api/v1/replay` endpoints + +--- + +## 13. Security Considerations + +1. **Key Management**: Signing keys managed by Authority service with rotation support +2. **Tenant Isolation**: Proofs scoped to tenants; cross-tenant access prohibited +3. **Integrity**: All digests use SHA-256; Merkle proofs enable partial verification +4. **Immutability**: Proofs cannot be modified once signed +5. **Audit**: All verification attempts logged with correlation IDs +6. **Air-gap**: Proofs and bundles can be exported for offline verification + +--- + +## 14. References + +- [DSSE Specification](https://github.com/secure-systems-lab/dsse) +- [RFC 8785 - JSON Canonicalization](https://tools.ietf.org/html/rfc8785) +- [in-toto Attestation Framework](https://github.com/in-toto/attestation) +- [SLSA Provenance](https://slsa.dev/provenance) +- [Platform Architecture](../platform/architecture-overview.md) +- [Facet Sealing Architecture](../facet/architecture.md) + +--- + +*Last updated: 2026-01-05* diff --git a/docs/modules/riskengine/architecture.md b/docs/modules/risk-engine/architecture.md similarity index 100% rename from docs/modules/riskengine/architecture.md rename to docs/modules/risk-engine/architecture.md diff --git a/docs/modules/sbomservice/README.md b/docs/modules/sbom-service/README.md similarity index 100% rename from docs/modules/sbomservice/README.md rename to docs/modules/sbom-service/README.md diff --git a/docs/modules/sbomservice/api/projection-read.md b/docs/modules/sbom-service/api/projection-read.md similarity index 100% rename from docs/modules/sbomservice/api/projection-read.md rename to docs/modules/sbom-service/api/projection-read.md diff --git a/docs/modules/sbomservice/architecture.md b/docs/modules/sbom-service/architecture.md similarity index 100% rename from docs/modules/sbomservice/architecture.md rename to docs/modules/sbom-service/architecture.md diff --git a/docs/modules/sbomservice/byos-ingestion.md b/docs/modules/sbom-service/byos-ingestion.md similarity index 100% rename from docs/modules/sbomservice/byos-ingestion.md rename to docs/modules/sbom-service/byos-ingestion.md diff --git a/docs/modules/sbomservice/fixtures/lnm-v1/README.md b/docs/modules/sbom-service/fixtures/lnm-v1/README.md similarity index 100% rename from docs/modules/sbomservice/fixtures/lnm-v1/README.md rename to docs/modules/sbom-service/fixtures/lnm-v1/README.md diff --git a/docs/modules/sbomservice/fixtures/lnm-v1/SHA256SUMS b/docs/modules/sbom-service/fixtures/lnm-v1/SHA256SUMS similarity index 100% rename from docs/modules/sbomservice/fixtures/lnm-v1/SHA256SUMS rename to docs/modules/sbom-service/fixtures/lnm-v1/SHA256SUMS diff --git a/docs/modules/sbomservice/fixtures/lnm-v1/catalog.json b/docs/modules/sbom-service/fixtures/lnm-v1/catalog.json similarity index 100% rename from docs/modules/sbomservice/fixtures/lnm-v1/catalog.json rename to docs/modules/sbom-service/fixtures/lnm-v1/catalog.json diff --git a/docs/modules/sbomservice/fixtures/lnm-v1/component_lookup.json b/docs/modules/sbom-service/fixtures/lnm-v1/component_lookup.json similarity index 100% rename from docs/modules/sbomservice/fixtures/lnm-v1/component_lookup.json rename to docs/modules/sbom-service/fixtures/lnm-v1/component_lookup.json diff --git a/docs/modules/sbomservice/fixtures/lnm-v1/projections.json b/docs/modules/sbom-service/fixtures/lnm-v1/projections.json similarity index 100% rename from docs/modules/sbomservice/fixtures/lnm-v1/projections.json rename to docs/modules/sbom-service/fixtures/lnm-v1/projections.json diff --git a/docs/modules/sbomservice/ledger-lineage.md b/docs/modules/sbom-service/ledger-lineage.md similarity index 100% rename from docs/modules/sbomservice/ledger-lineage.md rename to docs/modules/sbom-service/ledger-lineage.md diff --git a/docs/modules/sbomservice/lineage-ledger.md b/docs/modules/sbom-service/lineage-ledger.md similarity index 100% rename from docs/modules/sbomservice/lineage-ledger.md rename to docs/modules/sbom-service/lineage-ledger.md diff --git a/docs/modules/sbomservice/lineage/architecture.md b/docs/modules/sbom-service/lineage/architecture.md similarity index 100% rename from docs/modules/sbomservice/lineage/architecture.md rename to docs/modules/sbom-service/lineage/architecture.md diff --git a/docs/modules/sbomservice/lineage/schema.sql b/docs/modules/sbom-service/lineage/schema.sql similarity index 100% rename from docs/modules/sbomservice/lineage/schema.sql rename to docs/modules/sbom-service/lineage/schema.sql diff --git a/docs/modules/sbomservice/lineage/ui-architecture.md b/docs/modules/sbom-service/lineage/ui-architecture.md similarity index 100% rename from docs/modules/sbomservice/lineage/ui-architecture.md rename to docs/modules/sbom-service/lineage/ui-architecture.md diff --git a/docs/modules/sbomservice/offline-feed-plan.md b/docs/modules/sbom-service/offline-feed-plan.md similarity index 100% rename from docs/modules/sbomservice/offline-feed-plan.md rename to docs/modules/sbom-service/offline-feed-plan.md diff --git a/docs/modules/sbomservice/retention-policy.md b/docs/modules/sbom-service/retention-policy.md similarity index 100% rename from docs/modules/sbomservice/retention-policy.md rename to docs/modules/sbom-service/retention-policy.md diff --git a/docs/modules/sbomservice/reviews/2025-11-23-airgap-parity.md b/docs/modules/sbom-service/reviews/2025-11-23-airgap-parity.md similarity index 100% rename from docs/modules/sbomservice/reviews/2025-11-23-airgap-parity.md rename to docs/modules/sbom-service/reviews/2025-11-23-airgap-parity.md diff --git a/docs/modules/sbomservice/runbooks/airgap-parity-review.md b/docs/modules/sbom-service/runbooks/airgap-parity-review.md similarity index 100% rename from docs/modules/sbomservice/runbooks/airgap-parity-review.md rename to docs/modules/sbom-service/runbooks/airgap-parity-review.md diff --git a/docs/modules/sbomservice/sources/architecture.md b/docs/modules/sbom-service/sources/architecture.md similarity index 100% rename from docs/modules/sbomservice/sources/architecture.md rename to docs/modules/sbom-service/sources/architecture.md diff --git a/docs/modules/scanner/design/offline-kit-parity.md b/docs/modules/scanner/design/offline-kit-parity.md index 3e2ab6edb..18b64d9e5 100644 --- a/docs/modules/scanner/design/offline-kit-parity.md +++ b/docs/modules/scanner/design/offline-kit-parity.md @@ -320,4 +320,4 @@ When schemas/adapters change: - Sprint: `docs/implplan/SPRINT_0186_0001_0001_record_deterministic_execution.md` (SC10) - Roadmap: `docs/modules/scanner/design/standards-convergence-roadmap.md` (SC1) - Governance: `docs/modules/scanner/design/schema-governance.md` (SC9) -- Offline Operation: `docs/24_OFFLINE_KIT.md` +- Offline Operation: `docs/OFFLINE_KIT.md` diff --git a/docs/modules/scanner/guides/binary-evidence-guide.md b/docs/modules/scanner/guides/binary-evidence-guide.md index b28049886..77ee14c55 100644 --- a/docs/modules/scanner/guides/binary-evidence-guide.md +++ b/docs/modules/scanner/guides/binary-evidence-guide.md @@ -277,4 +277,4 @@ Stripped binaries may lack Build-IDs. Options: - [BinaryIndex Architecture](../../binaryindex/architecture.md) - [Scanner Architecture](../architecture.md) - [Proof Chain Specification](../../attestor/proof-chain-specification.md) -- [CLI Reference](../../../09_API_CLI_REFERENCE.md) +- [CLI Reference](../../../API_CLI_REFERENCE.md) diff --git a/docs/modules/scanner/guides/surface-fs-workflow.md b/docs/modules/scanner/guides/surface-fs-workflow.md index 445b4b27c..cf8d1346c 100644 --- a/docs/modules/scanner/guides/surface-fs-workflow.md +++ b/docs/modules/scanner/guides/surface-fs-workflow.md @@ -411,4 +411,4 @@ var payload = await _payloadStore.GetAsync(artifact.Uri, ct); - [Surface.FS Design](../design/surface-fs.md) - [Surface.Env Design](../design/surface-env.md) - [Surface.Validation Guide](./surface-validation-extensibility.md) -- [Offline Kit Documentation](../../../../24_OFFLINE_KIT.md) +- [Offline Kit Documentation](../../../../OFFLINE_KIT.md) diff --git a/docs/modules/scanner/operations/dsse-rekor-operator-guide.md b/docs/modules/scanner/operations/dsse-rekor-operator-guide.md index 92bfa12af..102e76d15 100644 --- a/docs/modules/scanner/operations/dsse-rekor-operator-guide.md +++ b/docs/modules/scanner/operations/dsse-rekor-operator-guide.md @@ -23,7 +23,7 @@ | Rekor v2 (managed or self-hosted) | Transparency log providing UUIDs + inclusion proofs. | `docs/ops/rekor/README.md` (if self-hosted) | | `StellaOps.Scanner` (WebService/Worker) | Requests attestations per scan, stores Rekor metadata next to SBOM artefacts. | `docs/modules/scanner/architecture.md` | | Export Center | Packages DSSE payloads + proofs into Offline Kit bundles and mirrors license notices. | `docs/modules/export-center/architecture.md` | -| Policy Engine + CLI | Enforce “attested only” promotion, expose CLI verification verbs. | `docs/modules/policy/architecture.md`, `docs/09_API_CLI_REFERENCE.md` | +| Policy Engine + CLI | Enforce “attested only” promotion, expose CLI verification verbs. | `docs/modules/policy/architecture.md`, `docs/API_CLI_REFERENCE.md` | --- @@ -210,4 +210,4 @@ stellaops-cli attest verify --envelope artifacts/scan123/attest/sbom.dsse.json \ - Scanner architecture (§Signer → Attestor → Rekor): `docs/modules/scanner/architecture.md` - Export Center profiles: `docs/modules/export-center/architecture.md` - Policy Engine predicates: `docs/modules/policy/architecture.md` -- CLI reference: `docs/09_API_CLI_REFERENCE.md` +- CLI reference: `docs/API_CLI_REFERENCE.md` diff --git a/docs/modules/scanner/operations/secret-leak-detection.md b/docs/modules/scanner/operations/secret-leak-detection.md index f393c724b..ac2460d49 100644 --- a/docs/modules/scanner/operations/secret-leak-detection.md +++ b/docs/modules/scanner/operations/secret-leak-detection.md @@ -371,5 +371,5 @@ The bundle was created without the `--sign` flag. Either: - `docs/modules/policy/secret-leak-detection-readiness.md` - `docs/benchmarks/scanner/deep-dives/secrets.md` - `docs/modules/scanner/design/surface-secrets.md` -- `docs/07_HIGH_LEVEL_ARCHITECTURE.md` - Runtime inventory (Scanner) +- `docs/ARCHITECTURE_OVERVIEW.md` - Runtime inventory (Scanner) - [Secrets Bundle Rotation](./secrets-bundle-rotation.md) diff --git a/docs/modules/sdk/README.md b/docs/modules/sdk/README.md index aa9345811..8262e3cb1 100644 --- a/docs/modules/sdk/README.md +++ b/docs/modules/sdk/README.md @@ -39,7 +39,7 @@ Key features: ## Related Documentation -- API Reference: `../../09_API_CLI_REFERENCE.md` +- API Reference: `../../API_CLI_REFERENCE.md` - OpenAPI Specs: `../../api/` (if exists) - CLI: `../cli/` - Gateway: `../gateway/` diff --git a/docs/modules/signals/README.md b/docs/modules/signals/README.md index 35b7885d2..19b145b5b 100644 --- a/docs/modules/signals/README.md +++ b/docs/modules/signals/README.md @@ -51,7 +51,7 @@ Key settings: - Architecture: `./architecture.md` - Policy Engine: `../policy/` - VexLens: `../vex-lens/` -- High-Level Architecture: `../../07_HIGH_LEVEL_ARCHITECTURE.md` +- High-Level Architecture: `../../ARCHITECTURE_OVERVIEW.md` ## Current Status diff --git a/docs/modules/snapshot/README.md b/docs/modules/snapshot/README.md index f57014d73..54654762d 100644 --- a/docs/modules/snapshot/README.md +++ b/docs/modules/snapshot/README.md @@ -44,7 +44,7 @@ Snapshot functionality is implemented across multiple modules: - AirGap: `../airgap/` - ExportCenter: `../export-center/` - Replay: `../replay/` (if exists) -- Offline Kit: `../../24_OFFLINE_KIT.md` +- Offline Kit: `../../OFFLINE_KIT.md` ## Current Status diff --git a/docs/modules/timelineindexer/architecture.md b/docs/modules/timeline-indexer/architecture.md similarity index 100% rename from docs/modules/timelineindexer/architecture.md rename to docs/modules/timeline-indexer/architecture.md diff --git a/docs/modules/ui/README.md b/docs/modules/ui/README.md index 1d944da00..60c067a5a 100644 --- a/docs/modules/ui/README.md +++ b/docs/modules/ui/README.md @@ -32,7 +32,7 @@ The Console presents operator dashboards for scans, policies, VEX evidence, runt - Auth smoke tests in `operations/auth-smoke.md`. - Observability runbook + dashboard stub in `operations/observability.md` and `operations/dashboards/console-ui-observability.json` (offline import). - Console architecture doc for layout and SSE fan-out. -- Operator guide: `../../15_UI_GUIDE.md`. Accessibility: `../../accessibility.md`. Security: `../../security/`. +- Operator guide: `../../UI_GUIDE.md`. Accessibility: `../../accessibility.md`. Security: `../../security/`. ## Related resources - ./operations/auth-smoke.md diff --git a/docs/modules/ui/console-architecture.md b/docs/modules/ui/console-architecture.md index f42ef35c1..805b4fffc 100644 --- a/docs/modules/ui/console-architecture.md +++ b/docs/modules/ui/console-architecture.md @@ -4,7 +4,7 @@ > **Ownership:** Console Guild • Docs Guild > **Delivery scope:** `StellaOps.Web` Angular workspace, Console Web Gateway routes (`/console/*`), Downloads manifest surfacing, SSE fan-out for Scheduler & telemetry. -> **Related docs:** [Console operator guide](../../15_UI_GUIDE.md), [Admin workflows](../../console/admin-tenants.md), [Air-gap workflows](../../console/airgap.md), [Console security posture](../../security/console-security.md), [Console observability](../../console/observability.md), [UI telemetry](../../observability/ui-telemetry.md), [Deployment guide](../../deploy/console.md) +> **Related docs:** [Console operator guide](../../UI_GUIDE.md), [Admin workflows](../../console/admin-tenants.md), [Air-gap workflows](../../console/airgap.md), [Console security posture](../../security/console-security.md), [Console observability](../../console/observability.md), [UI telemetry](../../observability/ui-telemetry.md), [Deployment guide](../../deploy/console.md) This dossier describes the end-to-end architecture of the StellaOps Console as delivered in Sprint 23. It covers the Angular workspace layout, API/gateway integration points, live-update channels, performance budgets, offline workflows, and observability hooks needed to keep the console deterministic and air-gap friendly. diff --git a/docs/modules/ui/wireframes/proof-visualization-wireframes.md b/docs/modules/ui/wireframes/proof-visualization-wireframes.md index 751f0d4e6..918c7854c 100644 --- a/docs/modules/ui/wireframes/proof-visualization-wireframes.md +++ b/docs/modules/ui/wireframes/proof-visualization-wireframes.md @@ -414,6 +414,6 @@ Deep-dive into the cryptographic attestation chain, showing DSSE envelopes and R ## References - `docs/db/SPECIFICATION.md` Section 5.6-5.8 — Schema definitions -- `docs/24_OFFLINE_KIT.md` Section 2.2 — Proof replay workflow +- `docs/OFFLINE_KIT.md` Section 2.2 — Proof replay workflow - `SPRINT_3500_0001_0001_deeper_moat_master.md` — Feature requirements - `docs/modules/ui/architecture.md` — Console architecture diff --git a/docs/modules/vexhub/README.md b/docs/modules/vex-hub/README.md similarity index 100% rename from docs/modules/vexhub/README.md rename to docs/modules/vex-hub/README.md diff --git a/docs/modules/vexhub/architecture.md b/docs/modules/vex-hub/architecture.md similarity index 100% rename from docs/modules/vexhub/architecture.md rename to docs/modules/vex-hub/architecture.md diff --git a/docs/modules/vexhub/integration-guide.md b/docs/modules/vex-hub/integration-guide.md similarity index 100% rename from docs/modules/vexhub/integration-guide.md rename to docs/modules/vex-hub/integration-guide.md diff --git a/docs/observability/observability.md b/docs/observability/observability.md index 51b79b0dc..ca6cd93fb 100644 --- a/docs/observability/observability.md +++ b/docs/observability/observability.md @@ -86,7 +86,7 @@ This guide captures the canonical signals emitted by Concelier and Excititor onc ### 2.2 Trace usage -- Correlate UI dashboard entries with traces via `traceId` surfaced in violation drawers (`docs/15_UI_GUIDE.md`). +- Correlate UI dashboard entries with traces via `traceId` surfaced in violation drawers (`docs/UI_GUIDE.md`). - Use `aoc.guard` spans to inspect guard payload snapshots. Sensitive fields are redacted automatically; raw JSON lives in secure logs only. - For scheduled verification, filter traces by `initiator="scheduled"` to compare runtimes pre/post change. diff --git a/docs/observability/ui-telemetry.md b/docs/observability/ui-telemetry.md index 298ac044f..a1730c1a0 100644 --- a/docs/observability/ui-telemetry.md +++ b/docs/observability/ui-telemetry.md @@ -78,7 +78,7 @@ - `ui.api.fetch` – HTTP fetch to backend; attributes: `service`, `endpoint`, `status`, `networkTime`. - `ui.sse.stream` – Server-sent event subscriptions (status ticker, runs); attributes: `channel`, `connectedMillis`, `reconnects`. - `ui.telemetry.batch` – Browser OTLP flush; attributes: `batchSize`, `success`, `retryCount`. - - `ui.policy.action` – Policy workspace actions (simulate, approve, activate) per `docs/15_UI_GUIDE.md`. + - `ui.policy.action` – Policy workspace actions (simulate, approve, activate) per `docs/UI_GUIDE.md`. - **Propagation:** Spans use W3C `traceparent`; gateway echoes header to backend APIs so traces stitch across UI → gateway → service. - **Sampling controls:** `OTEL_TRACES_SAMPLER_ARG` (ratio) and feature flag `telemetry.forceSampling` (sets to 100 % for incident debugging). - **Viewing traces:** Grafana Tempo or Jaeger via collector. Filter by `service.name = stellaops-console`. For cross-service debugging, filter on `correlationId` and `tenant`. @@ -147,7 +147,7 @@ Integrate alerts with Notifier (`ui.alerts`) or existing Ops channels. Tag incid | `OTEL_SERVICE_NAME` | Service tag for traces/logs. Set to `stellaops-console`. | auto | | `CONSOLE_TELEMETRY_SSE_ENABLED` | Enables `/console/telemetry` SSE feed for dashboards. | `true` | -Feature flag changes should be tracked in release notes and mirrored in `docs/15_UI_GUIDE.md` (navigation and workflow expectations). +Feature flag changes should be tracked in release notes and mirrored in `docs/UI_GUIDE.md` (navigation and workflow expectations). --- @@ -171,7 +171,7 @@ Feature flag changes should be tracked in release notes and mirrored in `docs/15 - [ ] DPoP/fresh-auth anomalies correlated with Authority audit logs during drill. - [ ] Offline capture workflow exercised; evidence stored in audit vault. - [ ] Screenshots of Grafana dashboards committed once they stabilise (update references). -- [ ] Cross-links verified (`docs/deploy/console.md`, `docs/security/console-security.md`, `docs/15_UI_GUIDE.md`). +- [ ] Cross-links verified (`docs/deploy/console.md`, `docs/security/console-security.md`, `docs/UI_GUIDE.md`). --- @@ -179,7 +179,7 @@ Feature flag changes should be tracked in release notes and mirrored in `docs/15 - `/docs/deploy/console.md` – Metrics endpoint, OTLP config, health checks. - `/docs/security/console-security.md` – Security metrics & alert hints. -- `docs/15_UI_GUIDE.md` – Console workflows and offline posture. +- `docs/UI_GUIDE.md` – Console workflows and offline posture. - `/docs/observability/observability.md` – Platform-wide practices. - `/ops/telemetry-collector.md` & `/ops/telemetry-storage.md` – Collector deployment. - `/docs/operations/console-docker-install.md` – Compose/Helm environment variables. diff --git a/docs/operations/configuration-guide.md b/docs/operations/configuration-guide.md index cd07c9563..67272cdcc 100644 --- a/docs/operations/configuration-guide.md +++ b/docs/operations/configuration-guide.md @@ -527,6 +527,6 @@ See `docs/operations/configuration-migration.md` for detailed migration steps. ## Related Documentation - [PostgreSQL Operations Guide](./postgresql-guide.md) -- [Air-Gap Deployment](../24_OFFLINE_KIT.md) +- [Air-Gap Deployment](../OFFLINE_KIT.md) - [Crypto Profile Reference](../modules/cryptography/architecture.md) - [Policy Engine Guide](../modules/policy/architecture.md) diff --git a/docs/operations/configuration-migration.md b/docs/operations/configuration-migration.md index 739ab99f7..c66a04f1d 100644 --- a/docs/operations/configuration-migration.md +++ b/docs/operations/configuration-migration.md @@ -229,5 +229,5 @@ docker compose down && docker compose up -d ## Related Documentation - [Configuration Guide](./configuration-guide.md) -- [Air-Gap Deployment](../24_OFFLINE_KIT.md) +- [Air-Gap Deployment](../OFFLINE_KIT.md) - [Docker Compose README](../../devops/compose/README.md) diff --git a/docs/operations/postgresql-guide.md b/docs/operations/postgresql-guide.md index 939fe5419..a060c2515 100644 --- a/docs/operations/postgresql-guide.md +++ b/docs/operations/postgresql-guide.md @@ -675,7 +675,7 @@ ALTER DATABASE stellaops SET default_transaction_read_only = on; ### 10.1 Offline Setup -PostgreSQL 16+ is bundled in the air-gap kit. See `docs/24_OFFLINE_KIT.md` for import instructions. +PostgreSQL 16+ is bundled in the air-gap kit. See `docs/OFFLINE_KIT.md` for import instructions. **Docker image digest (pinned):** ```yaml diff --git a/docs/operations/rekor-policy.md b/docs/operations/rekor-policy.md index 82682c957..154e1adb3 100644 --- a/docs/operations/rekor-policy.md +++ b/docs/operations/rekor-policy.md @@ -227,5 +227,5 @@ attestor: - [Attestor AGENTS.md](../../src/Attestor/StellaOps.Attestor/AGENTS.md) - [Scanner Score Proofs API](../api/scanner-score-proofs-api.md) -- [Offline Kit Specification](../24_OFFLINE_KIT.md) +- [Offline Kit Specification](../OFFLINE_KIT.md) - [Sigstore Rekor Documentation](https://docs.sigstore.dev/rekor/overview/) diff --git a/docs/policy/gateway.md b/docs/policy/gateway.md index e44466815..e658764db 100644 --- a/docs/policy/gateway.md +++ b/docs/policy/gateway.md @@ -123,7 +123,7 @@ curl -sS https://gateway.example.com/api/policy/packs/policy.core/revisions/5:ac -d '{"comment":"Rollout baseline"}' ``` -For air-gapped environments, bundle `policy-gateway.yaml` and the DPoP key in the Offline Kit (see `/docs/24_OFFLINE_KIT.md` §5.7). +For air-gapped environments, bundle `policy-gateway.yaml` and the DPoP key in the Offline Kit (see `/docs/OFFLINE_KIT.md` §5.7). > **DPoP proof helper:** Use `stella auth dpop proof` to mint sender-constrained proofs locally. The command accepts `--htu`, `--htm`, and `--token` arguments and emits a ready-to-use header value. Teams maintaining alternate tooling (for example, `scripts/make-dpop.sh`) can substitute it as long as the inputs and output match the CLI behaviour. diff --git a/docs/policy/vex-trust-model.md b/docs/policy/vex-trust-model.md index 974c151d8..9bf5c72db 100644 --- a/docs/policy/vex-trust-model.md +++ b/docs/policy/vex-trust-model.md @@ -12,7 +12,7 @@ Policy decisions about VEX rely on evidence produced by VEX ingestion and correl - **Consensus view** (when enabled) that summarizes the current effective status and conflicts - **Issuer registry / directory** that defines trust tiers and verification rules per tenant -See `docs/16_VEX_CONSENSUS_GUIDE.md` for the conceptual model and `docs/modules/excititor/architecture.md` + `docs/modules/vex-lens/architecture.md` for implementation details. +See `docs/VEX_CONSENSUS_GUIDE.md` for the conceptual model and `docs/modules/excititor/architecture.md` + `docs/modules/vex-lens/architecture.md` for implementation details. ## Principles @@ -48,6 +48,6 @@ Policy simulation is used to preview how changes in VEX trust settings or except ## References -- `docs/16_VEX_CONSENSUS_GUIDE.md` +- `docs/VEX_CONSENSUS_GUIDE.md` - `docs/modules/excititor/architecture.md` - `docs/modules/vex-lens/architecture.md` diff --git a/docs/product-advisories/05-Dec-2026 - New Testing Enhancements for Stella Ops.md b/docs/product-advisories/05-Dec-2026 - New Testing Enhancements for Stella Ops.md new file mode 100644 index 000000000..a3b289d5a --- /dev/null +++ b/docs/product-advisories/05-Dec-2026 - New Testing Enhancements for Stella Ops.md @@ -0,0 +1,85 @@ +**Stella Ops — Incremental Testing Enhancements (NEW since prior runs)** +*Only net-new ideas and practices; no restatement of earlier guidance.* + +--- + +## 1) Unit Testing — what to add now + +* **Semantic fuzzing for policies**: generate inputs that specifically target policy boundaries (quotas, geo rules, sanctions, priority overrides), not random fuzz. +* **Time-skew simulation**: unit tests that warp time (clock drift, leap seconds, TTL expiry) to catch cache and signature failures. +* **Decision explainability tests**: assert that every routing decision produces a minimal, machine-readable explanation payload (even if not user-facing). + +**Why it matters**: catches failures that only appear under temporal or policy edge conditions. + +--- + +## 2) Module / Source-Level Testing — new practices + +* **Policy-as-code tests**: treat routing and ops policies as versioned code with diff-based tests (policy change → expected behavior delta). +* **Schema evolution tests**: automatically replay last N schema versions against current code to ensure backward compatibility. +* **Dead-path detection**: fail builds if conditional branches are never exercised across the module test suite. + +**Why it matters**: prevents silent behavior changes when policies or schemas evolve. + +--- + +## 3) Integration Testing — new focus areas + +* **Production trace replay (sanitized)**: replay real, anonymized traces into integration environments to validate behavior against reality, not assumptions. +* **Failure choreography tests**: deliberately stagger dependency failures (A fails first, then B recovers, then A recovers) and assert system convergence. +* **Idempotency verification**: explicit tests that repeated requests under retries never create divergent state. + +**Why it matters**: most real outages are sequencing problems, not single failures. + +--- + +## 4) Deployment / E2E Testing — additions + +* **Config-diff E2E tests**: assert that changing *only* config (no code) produces only the expected behavioral delta. +* **Rollback lag tests**: measure and assert maximum time-to-safe-state after rollback is triggered. +* **Synthetic adversarial traffic**: continuously inject malformed but valid-looking traffic post-deploy to ensure defenses stay active. + +**Why it matters**: many incidents come from “safe” config changes and slow rollback propagation. + +--- + +## 5) Competitor Parity Testing — next-level + +* **Behavioral fingerprinting**: derive a compact fingerprint (outputs + timing + error shape) per request class and track drift over time. +* **Asymmetric stress tests**: apply load patterns competitors are known to struggle with and verify Stella Ops remains stable. +* **Regression-to-market alerts**: trigger alerts when Stella deviates from competitor norms in *either* direction (worse or suspiciously better). + +**Why it matters**: parity isn’t static; it drifts quietly unless measured continuously. + +--- + +## 6) New Cross-Cutting Standards to Enforce + +* **Tests as evidence**: every integration/E2E run produces immutable artifacts suitable for audit or post-incident review. +* **Deterministic replayability**: any failed test must be reproducible bit-for-bit within 24 hours. +* **Blast-radius annotation**: every test declares what operational surface it covers (routing, auth, billing, compliance). + +--- + +## Prioritized Checklist — This Week Only + +**Immediate (1–2 days)** + +1. Add decision-explainability assertions to core routing unit tests. +2. Introduce time-skew unit tests for cache, TTL, and signature logic. +3. Define and enforce idempotency tests on one critical integration path. + +**Short-term (by end of week)** +4. Enable sanitized production trace replay in one integration suite. +5. Add rollback lag measurement to deployment/E2E tests. +6. Start policy-as-code diff tests for routing rules. + +**High-leverage** +7. Implement a minimal competitor behavioral fingerprint and store it weekly. +8. Require blast-radius annotations on all new integration and E2E tests. + +--- + +### Bottom line + +The next gains for Stella Ops testing are no longer about coverage—they’re about **temporal correctness, policy drift control, replayability, and competitive awareness**. Systems that fail now do so quietly, over time, and under sequence pressure. These additions close exactly those gaps. diff --git a/docs/quickstart.md b/docs/quickstart.md index 8a4545ab9..4b2a1ecb6 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -32,7 +32,7 @@ cosign verify-blob \ docker-compose.stella-ops.yml ``` -*Air-gapped?* The [Offline Update Kit](24_OFFLINE_KIT.md) ships these files plus feeds and plug-ins. +*Air-gapped?* The [Offline Update Kit](OFFLINE_KIT.md) ships these files plus feeds and plug-ins. ## 2. Configure `.env` (1 min) @@ -93,6 +93,6 @@ stella scan image \ ### Next steps -- Harden the deployment with [`17_SECURITY_HARDENING_GUIDE.md`](17_SECURITY_HARDENING_GUIDE.md). +- Harden the deployment with [SECURITY_HARDENING_GUIDE.md](SECURITY_HARDENING_GUIDE.md). - Explore feature highlights in [`key-features.md`](key-features.md). - Plan the rollout using the [evaluation checklist](onboarding/evaluation-checklist.md). diff --git a/docs/reachability/graph-revision-schema.md b/docs/reachability/graph-revision-schema.md index c1ee507fc..ca0201afa 100644 --- a/docs/reachability/graph-revision-schema.md +++ b/docs/reachability/graph-revision-schema.md @@ -370,7 +370,7 @@ Authorization: Bearer - [richgraph-v1 Contract](../contracts/richgraph-v1.md) - Graph schema specification - [Function-Level Evidence](./function-level-evidence.md) - Evidence chain guide - [CAS Infrastructure](../contracts/cas-infrastructure.md) - Content-addressable storage -- [Offline Kit](../24_OFFLINE_KIT.md) - Air-gap deployment +- [Offline Kit](../OFFLINE_KIT.md) - Air-gap deployment --- diff --git a/docs/reproducibility.md b/docs/reproducibility.md index 38420249f..189ed50e3 100644 --- a/docs/reproducibility.md +++ b/docs/reproducibility.md @@ -321,7 +321,7 @@ Policy thresholds are attested in verdict bundles: - [Testing Strategy](testing/testing-strategy-models.md) - [Determinism Verification](testing/determinism-verification.md) - [DSSE Attestation Guide](modules/attestor/README.md) -- [Offline Operation](24_OFFLINE_KIT.md) +- [Offline Operation](OFFLINE_KIT.md) - [Proof Bundle Spec](modules/triage/proof-bundle-spec.md) ## Changelog diff --git a/docs/roadmap/README.md b/docs/roadmap/README.md index 83be9f2fc..da1e0a467 100644 --- a/docs/roadmap/README.md +++ b/docs/roadmap/README.md @@ -9,7 +9,7 @@ Scheduling and staffing live outside the documentation layer; this roadmap stays ## Canonical references by area - Architecture overview: `docs/40_ARCHITECTURE_OVERVIEW.md` -- High-level architecture: `docs/07_HIGH_LEVEL_ARCHITECTURE.md` -- Offline posture and workflows: `docs/24_OFFLINE_KIT.md`, `docs/airgap/overview.md` +- High-level architecture: `docs/ARCHITECTURE_OVERVIEW.md` +- Offline posture and workflows: `docs/OFFLINE_KIT.md`, `docs/airgap/overview.md` - Determinism principles: `docs/key-features.md`, `docs/testing/connector-fixture-discipline.md` - Security boundaries and roles: `docs/security/scopes-and-roles.md`, `docs/security/tenancy-overview.md` diff --git a/docs/roadmap/maturity-model.md b/docs/roadmap/maturity-model.md index e20fff0cd..702c3f52f 100644 --- a/docs/roadmap/maturity-model.md +++ b/docs/roadmap/maturity-model.md @@ -51,7 +51,7 @@ This document defines what "shipped" means for StellaOps capabilities. Each area | Level | What exists | Minimum evidence | | --- | --- | --- | -| Foundation | Documented offline concepts and supported workflows. | `docs/24_OFFLINE_KIT.md` plus importer/controller docs and examples. | +| Foundation | Documented offline concepts and supported workflows. | `docs/OFFLINE_KIT.md` plus importer/controller docs and examples. | | Hardened | Deterministic imports and verified indexes. | Byte-stable indexes with reproducible hash outputs across machines. | | Sovereign | Independent trust anchors and mirrors. | Trust-root provisioning docs and an air-gapped "day-2 ops" runbook. | | Ecosystem | Third-party bundles and toolchain integrations. | Conformance tests and offline bundle validation tooling. | diff --git a/docs/runtime/SCANNER_RUNTIME_READINESS.md b/docs/runtime/SCANNER_RUNTIME_READINESS.md index 56c37c4ca..559e6bfb9 100644 --- a/docs/runtime/SCANNER_RUNTIME_READINESS.md +++ b/docs/runtime/SCANNER_RUNTIME_READINESS.md @@ -34,7 +34,7 @@ This runbook confirms that Scanner.WebService now surfaces the metadata Runtime ``` (Use `npm install --no-save ajv ajv-cli ajv-formats` once per clone.) -> Snapshot fixtures: see `docs/events/samples/scanner.event.report.ready@1.sample.json` for a canonical orchestrator event that already carries `quietedFindingCount`. +> Snapshot fixtures: see `docs/events/samples/scanner.event.report.ready@1.sample.json` for a canonical orchestrator event that already carries `quietedFindingCount`. --- @@ -66,7 +66,7 @@ Scanner streams structured progress messages for each scan. The `data` map insid ``` Subsequent frames include additional hints as analyzers progress (e.g., `stage`, `meta.*`, or analyzer-provided keys). Ensure newline-delimited JSON consumers preserve the `data` dictionary when forwarding to runtime dashboards. -> The same frame structure is documented in `docs/09_API_CLI_REFERENCE.md` §2.6. Copy that snippet into integration tests to keep compatibility. +> The same frame structure is documented in `docs/API_CLI_REFERENCE.md` §2.6. Copy that snippet into integration tests to keep compatibility. --- diff --git a/docs/specs/symbols/SYMBOL_MANIFEST_v1.md b/docs/specs/symbols/SYMBOL_MANIFEST_v1.md index adc45f755..ccde1d3a1 100644 --- a/docs/specs/symbols/SYMBOL_MANIFEST_v1.md +++ b/docs/specs/symbols/SYMBOL_MANIFEST_v1.md @@ -75,6 +75,6 @@ Constraints: - Breaking changes will bump to `SYMBOL_MANIFEST/v2`. ## 8. References -- `docs/24_OFFLINE_KIT.md` (debug store expectations) +- `docs/OFFLINE_KIT.md` (debug store expectations) - `docs/benchmarks/signals/bench-determinism.md` - `docs/modules/scanner/architecture.md` (reachability + symbol linkage) diff --git a/docs/technical/architecture/data-flows.md b/docs/technical/architecture/data-flows.md index 9d462242b..71d632032 100644 --- a/docs/technical/architecture/data-flows.md +++ b/docs/technical/architecture/data-flows.md @@ -547,4 +547,4 @@ policy:evaluated notification.sent event indexed - [Module Matrix](module-matrix.md) - [Schema Mapping](schema-mapping.md) - [Data Schemas](../../11_DATA_SCHEMAS.md) -- [Offline Kit](../../24_OFFLINE_KIT.md) +- [Offline Kit](../../OFFLINE_KIT.md) diff --git a/docs/technical/architecture/request-flows.md b/docs/technical/architecture/request-flows.md index 8fd02bea5..4b025d339 100644 --- a/docs/technical/architecture/request-flows.md +++ b/docs/technical/architecture/request-flows.md @@ -17,7 +17,7 @@ This document describes the canonical end-to-end flows at a level useful for deb 11. **Scanner.WebService -> events stream**: publish completion events for notifications and downstream consumers. 12. **Notification engine -> channels**: render and deliver notifications with idempotency tracking. -Offline note: for air-gapped deployments, step 6 writes to local object storage and step 7 relies on offline mirrors/bundles rather than public feeds. See `docs/24_OFFLINE_KIT.md` and `docs/airgap/overview.md`. +Offline note: for air-gapped deployments, step 6 writes to local object storage and step 7 relies on offline mirrors/bundles rather than public feeds. See `docs/OFFLINE_KIT.md` and `docs/airgap/overview.md`. ### Scan execution sequence diagram diff --git a/docs/technical/architecture/user-flows.md b/docs/technical/architecture/user-flows.md index fa71d270f..556860b37 100644 --- a/docs/technical/architecture/user-flows.md +++ b/docs/technical/architecture/user-flows.md @@ -571,7 +571,7 @@ User requests export ## Related Documentation - [Architecture Overview](../../40_ARCHITECTURE_OVERVIEW.md) -- [High-Level Architecture](../../07_HIGH_LEVEL_ARCHITECTURE.md) +- [High-Level Architecture](../../ARCHITECTURE_OVERVIEW.md) - [Data Flows](data-flows.md) - [Schema Mapping](schema-mapping.md) - [Module Matrix](module-matrix.md) diff --git a/docs/technical/interfaces/README.md b/docs/technical/interfaces/README.md index 85c649e8f..29bd4ec46 100644 --- a/docs/technical/interfaces/README.md +++ b/docs/technical/interfaces/README.md @@ -3,7 +3,7 @@ Specifications covering APIs, data contracts, event envelopes, and enforcement models. ## External & Internal APIs -- [../09_API_CLI_REFERENCE.md](../../09_API_CLI_REFERENCE.md) – canonical REST and CLI surface (scan, policy, auth, health). +- [../API_CLI_REFERENCE.md](../../API_CLI_REFERENCE.md) – canonical REST and CLI surface (scan, policy, auth, health). - [../api/policy.md](../../api/policy.md) – Policy Engine REST endpoints. - Module APIs: see relevant module architecture docs (e.g., [../../modules/export-center/api.md](../../modules/export-center/api.md)). diff --git a/docs/vex/aggregation.md b/docs/vex/aggregation.md index 199ca615e..7b9dbf1dd 100644 --- a/docs/vex/aggregation.md +++ b/docs/vex/aggregation.md @@ -38,6 +38,6 @@ Downstream consumers (Policy/Console/Exports) use linksets to explain what disag ## References -- `docs/16_VEX_CONSENSUS_GUIDE.md` +- `docs/VEX_CONSENSUS_GUIDE.md` - `docs/modules/excititor/architecture.md` - `docs/modules/vex-lens/architecture.md` diff --git a/docs/vex/consensus-console.md b/docs/vex/consensus-console.md index 31443b154..bc463cf07 100644 --- a/docs/vex/consensus-console.md +++ b/docs/vex/consensus-console.md @@ -19,5 +19,5 @@ Every displayed fact should link back to: ## References -- Operator guide: `docs/15_UI_GUIDE.md` -- VEX conceptual guide: `docs/16_VEX_CONSENSUS_GUIDE.md` +- Operator guide: `docs/UI_GUIDE.md` +- VEX conceptual guide: `docs/VEX_CONSENSUS_GUIDE.md` diff --git a/docs/vex/consensus-overview.md b/docs/vex/consensus-overview.md index c95120e72..374235542 100644 --- a/docs/vex/consensus-overview.md +++ b/docs/vex/consensus-overview.md @@ -1,6 +1,6 @@ # VEX Evidence and Consensus (Detailed) -This document complements `docs/16_VEX_CONSENSUS_GUIDE.md` with implementation-oriented detail: what objects exist, how evidence is correlated without rewriting sources, and what “consensus” means in practice. +This document complements `docs/VEX_CONSENSUS_GUIDE.md` with implementation-oriented detail: what objects exist, how evidence is correlated without rewriting sources, and what "consensus" means in practice. ## Pipeline (Evidence First) @@ -27,5 +27,5 @@ This document complements `docs/16_VEX_CONSENSUS_GUIDE.md` with implementation-o - Ingestion, raw store, and linksets: `docs/modules/excititor/architecture.md` - Consensus and issuer trust: `docs/modules/vex-lens/architecture.md` -- Console/operator view: `docs/15_UI_GUIDE.md` -- Triage model: `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` +- Console/operator view: `docs/UI_GUIDE.md` +- Triage model: `docs/VULNERABILITY_EXPLORER_GUIDE.md` diff --git a/docs/vex/explorer-integration.md b/docs/vex/explorer-integration.md index f1f87ce4a..06312ea09 100644 --- a/docs/vex/explorer-integration.md +++ b/docs/vex/explorer-integration.md @@ -20,6 +20,6 @@ The Explorer must remain “quiet by default, never silent”: VEX-based suppres ## References -- `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` -- `docs/16_VEX_CONSENSUS_GUIDE.md` +- `docs/VULNERABILITY_EXPLORER_GUIDE.md` +- `docs/VEX_CONSENSUS_GUIDE.md` - `docs/modules/vuln-explorer/architecture.md` diff --git a/docs/vex/issuer-directory.md b/docs/vex/issuer-directory.md index 754bcc26c..e4899d19f 100644 --- a/docs/vex/issuer-directory.md +++ b/docs/vex/issuer-directory.md @@ -28,6 +28,6 @@ Offline deployments must be able to verify issuer identity without network acces ## References -- `docs/16_VEX_CONSENSUS_GUIDE.md` +- `docs/VEX_CONSENSUS_GUIDE.md` - `docs/modules/excititor/architecture.md` - `docs/modules/vex-lens/architecture.md` diff --git a/docs/vuln/explorer-overview.md b/docs/vuln/explorer-overview.md index e10249f52..85fe627bb 100644 --- a/docs/vuln/explorer-overview.md +++ b/docs/vuln/explorer-overview.md @@ -2,7 +2,7 @@ The Vulnerability Explorer is the evidence-linked triage surface that brings together SBOM facts, advisory/VEX evidence, reachability signals, policy explainability, and operator decisions into a single auditable workflow. -This document complements the high-level guide `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` with additional detail and cross-links. +This document complements the high-level guide `docs/VULNERABILITY_EXPLORER_GUIDE.md` with additional detail and cross-links. ## Core Objects @@ -20,6 +20,6 @@ This document complements the high-level guide `docs/20_VULNERABILITY_EXPLORER_G ## References -- High-level guide: `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` -- Console operator guide: `docs/15_UI_GUIDE.md` +- High-level guide: `docs/VULNERABILITY_EXPLORER_GUIDE.md` +- Console operator guide: `docs/UI_GUIDE.md` - Module dossier: `docs/modules/vuln-explorer/architecture.md` diff --git a/docs/vuln/explorer-using-console.md b/docs/vuln/explorer-using-console.md index 195d26a6a..f37315ecc 100644 --- a/docs/vuln/explorer-using-console.md +++ b/docs/vuln/explorer-using-console.md @@ -27,6 +27,6 @@ This document describes the operator workflow for triaging findings in the Conso ## References -- Console operator guide: `docs/15_UI_GUIDE.md` -- Vulnerability Explorer guide: `docs/20_VULNERABILITY_EXPLORER_GUIDE.md` -- Offline Kit: `docs/24_OFFLINE_KIT.md` +- Console operator guide: `docs/UI_GUIDE.md` +- Vulnerability Explorer guide: `docs/VULNERABILITY_EXPLORER_GUIDE.md` +- Offline Kit: `docs/OFFLINE_KIT.md` diff --git a/fix-asynclifetime.ps1 b/fix-asynclifetime.ps1 deleted file mode 100644 index ce623c662..000000000 --- a/fix-asynclifetime.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -Get-ChildItem -Path "src" -Filter "*.cs" -Recurse | ForEach-Object { - $content = Get-Content $_.FullName -Raw -ErrorAction SilentlyContinue - if ($content -match 'IAsyncLifetime') { - $content = $content -replace 'public Task InitializeAsync\(\)', 'public ValueTask InitializeAsync()' - $content = $content -replace 'public Task DisposeAsync\(\)', 'public ValueTask DisposeAsync()' - $content = $content -replace '=> Task\.CompletedTask;', '=> ValueTask.CompletedTask;' - Set-Content $_.FullName $content -NoNewline - Write-Host "Fixed: $($_.FullName)" - } -} diff --git a/fix-fluentassertions.ps1 b/fix-fluentassertions.ps1 deleted file mode 100644 index d4360ada0..000000000 --- a/fix-fluentassertions.ps1 +++ /dev/null @@ -1,10 +0,0 @@ -Get-ChildItem -Path "src" -Filter "*.cs" -Recurse | ForEach-Object { - $content = Get-Content $_.FullName -Raw -ErrorAction SilentlyContinue - $original = $content - $content = $content -replace '\.BeLessOrEqualTo\(', '.BeLessThanOrEqualTo(' - $content = $content -replace '\.BeGreaterOrEqualTo\(', '.BeGreaterThanOrEqualTo(' - if ($content -ne $original) { - Set-Content $_.FullName $content -NoNewline - Write-Host "Fixed: $($_.FullName)" - } -} diff --git a/fix-npgsql.ps1 b/fix-npgsql.ps1 deleted file mode 100644 index 4072e4619..000000000 --- a/fix-npgsql.ps1 +++ /dev/null @@ -1,8 +0,0 @@ -Get-ChildItem -Path "src" -Filter "*.csproj" -Recurse | ForEach-Object { - $content = Get-Content $_.FullName -Raw -ErrorAction SilentlyContinue - if ($content -match 'Npgsql\.EntityFrameworkCore\.PostgreSQL.*10\.0\.1') { - $content = $content -replace 'Npgsql\.EntityFrameworkCore\.PostgreSQL" Version="10\.0\.1"', 'Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.0"' - Set-Content $_.FullName $content -NoNewline - Write-Host "Fixed: $($_.FullName)" - } -} diff --git a/fix-xunit-refs.ps1 b/fix-xunit-refs.ps1 deleted file mode 100644 index 000f4c8a8..000000000 --- a/fix-xunit-refs.ps1 +++ /dev/null @@ -1,17 +0,0 @@ -$xunitPackages = @' - - - -'@ - -Get-ChildItem -Path "src" -Filter "*.csproj" -Recurse | Where-Object { $_.Name -like "*.Tests.csproj" } | ForEach-Object { - $content = Get-Content $_.FullName -Raw - if ($content -match 'true' -and $content -notmatch 'Include="xunit"') { - # Find the first ItemGroup with PackageReference and add xunit there - if ($content -match '(\s*\s* static () => new BunLanguageAnalyzer(), - "java" => static () => new JavaLanguageAnalyzer(), - "go" => static () => new GoLanguageAnalyzer(), - "node" => static () => new NodeLanguageAnalyzer(), - "dotnet" => static () => new DotNetLanguageAnalyzer(), - "python" => static () => new PythonLanguageAnalyzer(), - _ => throw new InvalidOperationException($"Unsupported analyzer '{analyzerId}'."), + // Note: Language-specific analyzers (Bun, Java, Go, Node, DotNet, Python) are loaded via plugin system. + // Benchmarks should use plugin-loaded analyzers instead of hardcoded references. + // See LanguageAnalyzerPluginCatalog for dynamic analyzer loading. + _ => throw new InvalidOperationException($"Unsupported analyzer '{analyzerId}'. Language analyzers must be loaded via plugin system."), }; } } diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj index 87d040caf..416546e47 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj @@ -8,6 +8,10 @@ true + + + + diff --git a/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs b/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs index e800d8c48..a52053d7d 100644 --- a/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs +++ b/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs @@ -69,9 +69,19 @@ internal static partial class CommandHandlers validator, builderLogger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + // Enumerate rule files from source directory + var ruleFiles = Directory.EnumerateFiles(sources, "*.json", SearchOption.AllDirectories) + .ToList(); + + if (ruleFiles.Count == 0) + { + AnsiConsole.MarkupLine($"[red]Error: No rule files (*.json) found in {Markup.Escape(sources)}[/]"); + return 1; + } + var buildOptions = new BundleBuildOptions { - SourceDirectory = sources, + RuleFiles = ruleFiles, OutputDirectory = output, BundleId = id, Version = bundleVersion, diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/Program.cs b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/Program.cs index b79d93c22..e527c6ed5 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/Program.cs +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/Program.cs @@ -71,3 +71,6 @@ builder.Services.AddScoped() builder.Services.AddHostedService(); await builder.Build().RunAsync().ConfigureAwait(false); + +// Explicit internal Program class to avoid conflicts with other projects that reference this assembly +internal sealed partial class Program { } diff --git a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj index 3d032bd4d..6240d1fa2 100644 --- a/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj +++ b/src/Notifier/StellaOps.Notifier/StellaOps.Notifier.Worker/StellaOps.Notifier.Worker.csproj @@ -1,4 +1,4 @@ - + net10.0 preview @@ -7,6 +7,10 @@ true + + + + diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/BudgetEnforcementIntegrationTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/BudgetEnforcementIntegrationTests.cs index 8f07d8992..0f65e7f80 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/BudgetEnforcementIntegrationTests.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/BudgetEnforcementIntegrationTests.cs @@ -20,7 +20,7 @@ public sealed class BudgetEnforcementIntegrationTests public BudgetEnforcementIntegrationTests() { - _ledger = new BudgetLedger(_store, NullLogger.Instance); + _ledger = new BudgetLedger(_store, logger: NullLogger.Instance); } #region Window Management Tests diff --git a/src/Scanner/StellaOps.Scanner.Worker/Metrics/ScanMetricsCollector.cs b/src/Scanner/StellaOps.Scanner.Worker/Metrics/ScanMetricsCollector.cs index 8111fb15e..73cbf517b 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Metrics/ScanMetricsCollector.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Metrics/ScanMetricsCollector.cs @@ -7,7 +7,7 @@ using System.Diagnostics; using Microsoft.Extensions.Logging; -using StellaOps.Determinism.Abstractions; +using StellaOps.Determinism; using StellaOps.Scanner.Storage.Models; using StellaOps.Scanner.Storage.Repositories; diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs index d66ada607..d6d6e23ea 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs @@ -26,14 +26,14 @@ internal sealed class SecretsAnalyzerStageExecutor : IScanStageExecutor "scanner.rootfs", }; - private readonly ISecretsAnalyzer _secretsAnalyzer; + private readonly SecretsAnalyzer _secretsAnalyzer; private readonly ScannerWorkerMetrics _metrics; private readonly TimeProvider _timeProvider; private readonly IOptions _options; private readonly ILogger _logger; public SecretsAnalyzerStageExecutor( - ISecretsAnalyzer secretsAnalyzer, + SecretsAnalyzer secretsAnalyzer, ScannerWorkerMetrics metrics, TimeProvider timeProvider, IOptions options, diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs index cfbb94986..8450411cd 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs @@ -42,6 +42,11 @@ public sealed class SecretsAnalyzer : ILanguageAnalyzer /// public SecretRuleset? Ruleset => _ruleset; + /// + /// Gets the ruleset version string for tracking and reporting. + /// + public string RulesetVersion => _ruleset?.Version ?? "unknown"; + /// /// Sets the ruleset to use for detection. /// Called by SecretsAnalyzerHost after loading the bundle. @@ -51,6 +56,58 @@ public sealed class SecretsAnalyzer : ILanguageAnalyzer _ruleset = ruleset ?? throw new ArgumentNullException(nameof(ruleset)); } + /// + /// Analyzes raw file content for secrets. Adapter for Worker stage executor. + /// + public async ValueTask> AnalyzeAsync( + byte[] content, + string relativePath, + CancellationToken ct) + { + if (!IsEnabled || content is null || content.Length == 0) + { + return new List(); + } + + var findings = new List(); + + foreach (var rule in _ruleset!.GetRulesForFile(relativePath)) + { + ct.ThrowIfCancellationRequested(); + + var matches = await _detector.DetectAsync(content, relativePath, rule, ct); + + foreach (var match in matches) + { + var confidence = MapScoreToConfidence(match.ConfidenceScore); + if (confidence < _options.Value.MinConfidence) + { + continue; + } + + var maskedSecret = _masker.Mask(match.Secret); + var finding = new SecretFinding + { + RuleId = rule.Id, + RuleName = rule.Name, + Severity = rule.Severity, + Confidence = confidence, + FilePath = relativePath, + LineNumber = match.LineNumber, + ColumnStart = match.ColumnStart, + ColumnEnd = match.ColumnEnd, + MatchedText = maskedSecret, + Category = rule.Category, + DetectedAtUtc = _timeProvider.GetUtcNow() + }; + + findings.Add(finding); + } + } + + return findings; + } + public async ValueTask AnalyzeAsync( LanguageAnalyzerContext context, LanguageComponentWriter writer, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageQueryPerformanceTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageQueryPerformanceTests.cs index cf47832d0..e83c65a71 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageQueryPerformanceTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageQueryPerformanceTests.cs @@ -169,7 +169,8 @@ public sealed class TriageQueryPerformanceTests : IAsyncLifetime CveId = i % 3 == 0 ? $"CVE-2021-{23337 + i}" : null, RuleId = i % 3 != 0 ? $"RULE-{i:D4}" : null, FirstSeenAt = DateTimeOffset.UtcNow.AddDays(-i), - LastSeenAt = DateTimeOffset.UtcNow.AddHours(-i) + LastSeenAt = DateTimeOffset.UtcNow.AddHours(-i), + UpdatedAt = DateTimeOffset.UtcNow.AddHours(-i) }; findings.Add(finding); } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageSchemaIntegrationTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageSchemaIntegrationTests.cs index 17b4e358c..7942e0ea4 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageSchemaIntegrationTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Triage.Tests/TriageSchemaIntegrationTests.cs @@ -60,6 +60,7 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime // Arrange await Context.Database.EnsureCreatedAsync(); + var now = DateTimeOffset.UtcNow; var finding = new TriageFinding { Id = Guid.NewGuid(), @@ -67,8 +68,9 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime AssetLabel = "prod/api-gateway:1.2.3", Purl = "pkg:npm/lodash@4.17.20", CveId = "CVE-2021-23337", - FirstSeenAt = DateTimeOffset.UtcNow, - LastSeenAt = DateTimeOffset.UtcNow + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; // Act @@ -90,13 +92,17 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime // Arrange await Context.Database.EnsureCreatedAsync(); + var now = DateTimeOffset.UtcNow; var finding = new TriageFinding { Id = Guid.NewGuid(), AssetId = Guid.NewGuid(), AssetLabel = "prod/api-gateway:1.2.3", Purl = "pkg:npm/lodash@4.17.20", - CveId = "CVE-2021-23337" + CveId = "CVE-2021-23337", + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; Context.Findings.Add(finding); @@ -111,7 +117,7 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime Note = "Code path is not reachable per RichGraph analysis", ActorSubject = "user:test@example.com", ActorDisplay = "Test User", - CreatedAt = DateTimeOffset.UtcNow + CreatedAt = now }; // Act @@ -137,13 +143,17 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime // Arrange await Context.Database.EnsureCreatedAsync(); + var now = DateTimeOffset.UtcNow; var finding = new TriageFinding { Id = Guid.NewGuid(), AssetId = Guid.NewGuid(), AssetLabel = "prod/api-gateway:1.2.3", Purl = "pkg:npm/lodash@4.17.20", - CveId = "CVE-2021-23337" + CveId = "CVE-2021-23337", + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; Context.Findings.Add(finding); @@ -160,7 +170,7 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime Verdict = TriageVerdict.Block, Lane = TriageLane.Blocked, Why = "High-severity CVE with network exposure", - ComputedAt = DateTimeOffset.UtcNow + ComputedAt = now }; // Act @@ -186,13 +196,17 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime // Arrange await Context.Database.EnsureCreatedAsync(); + var now = DateTimeOffset.UtcNow; var finding = new TriageFinding { Id = Guid.NewGuid(), AssetId = Guid.NewGuid(), AssetLabel = "prod/api:1.0", Purl = "pkg:npm/test@1.0.0", - CveId = "CVE-2024-0001" + CveId = "CVE-2024-0001", + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; Context.Findings.Add(finding); @@ -200,20 +214,24 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime var decision = new TriageDecision { + Id = Guid.NewGuid(), FindingId = finding.Id, Kind = TriageDecisionKind.Ack, ReasonCode = "ACKNOWLEDGED", - ActorSubject = "user:admin" + ActorSubject = "user:admin", + CreatedAt = now }; var riskResult = new TriageRiskResult { + Id = Guid.NewGuid(), FindingId = finding.Id, PolicyId = "policy-v1", PolicyVersion = "1.0", InputsHash = "hash123", Score = 50, - Why = "Medium risk" + Why = "Medium risk", + ComputedAt = now }; Context.Decisions.Add(decision); @@ -245,13 +263,18 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime const string purl = "pkg:npm/lodash@4.17.20"; const string cveId = "CVE-2021-23337"; + var now = DateTimeOffset.UtcNow; var finding1 = new TriageFinding { + Id = Guid.NewGuid(), AssetId = assetId, EnvironmentId = envId, AssetLabel = "prod/api:1.0", Purl = purl, - CveId = cveId + CveId = cveId, + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; Context.Findings.Add(finding1); @@ -259,11 +282,15 @@ public sealed class TriageSchemaIntegrationTests : IAsyncLifetime var finding2 = new TriageFinding { + Id = Guid.NewGuid(), AssetId = assetId, EnvironmentId = envId, AssetLabel = "prod/api:1.0", Purl = purl, - CveId = cveId + CveId = cveId, + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; Context.Findings.Add(finding2); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/FindingsEvidenceControllerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/FindingsEvidenceControllerTests.cs index 3a74afd7c..8e7c433ee 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/FindingsEvidenceControllerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/FindingsEvidenceControllerTests.cs @@ -132,6 +132,7 @@ public sealed class FindingsEvidenceControllerTests await db.Database.EnsureCreatedAsync(); + var now = DateTimeOffset.UtcNow; var findingId = Guid.NewGuid(); var finding = new TriageFinding { @@ -140,12 +141,15 @@ public sealed class FindingsEvidenceControllerTests AssetLabel = "prod/api-gateway:1.2.3", Purl = "pkg:npm/lodash@4.17.20", CveId = "CVE-2024-12345", - LastSeenAt = DateTimeOffset.UtcNow + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; db.Findings.Add(finding); db.RiskResults.Add(new TriageRiskResult { + Id = Guid.NewGuid(), FindingId = findingId, PolicyId = "policy-1", PolicyVersion = "1.0.0", @@ -154,15 +158,17 @@ public sealed class FindingsEvidenceControllerTests Verdict = TriageVerdict.Block, Lane = TriageLane.Blocked, Why = "High risk score", - ComputedAt = DateTimeOffset.UtcNow + ComputedAt = now }); db.EvidenceArtifacts.Add(new TriageEvidenceArtifact { + Id = Guid.NewGuid(), FindingId = findingId, Type = TriageEvidenceType.Provenance, Title = "SBOM attestation", ContentHash = "sha256:attestation", - Uri = "s3://evidence/attestation.json" + Uri = "s3://evidence/attestation.json", + CreatedAt = now }); await db.SaveChangesAsync(); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/GatingReasonServiceTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/GatingReasonServiceTests.cs index 5873f1cda..11b1fa3bc 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/GatingReasonServiceTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/GatingReasonServiceTests.cs @@ -448,6 +448,7 @@ public sealed class GatingReasonServiceTests public void VexEvidenceTrust_SignedWithLedger_HasHighTrust() { // Arrange - DSSE envelope + signature ref + source ref + var now = DateTimeOffset.UtcNow; var vex = new TriageEffectiveVex { Id = Guid.NewGuid(), @@ -455,7 +456,9 @@ public sealed class GatingReasonServiceTests DsseEnvelopeHash = "sha256:signed", SignatureRef = "ledger-entry", SourceDomain = "nvd", - SourceRef = "NVD-CVE-2024-1234" + SourceRef = "NVD-CVE-2024-1234", + ValidFrom = now, + CollectedAt = now }; // Assert - all evidence factors present @@ -469,6 +472,7 @@ public sealed class GatingReasonServiceTests public void VexEvidenceTrust_NoEvidence_HasBaseTrust() { // Arrange - no signature, no ledger, no source + var now = DateTimeOffset.UtcNow; var vex = new TriageEffectiveVex { Id = Guid.NewGuid(), @@ -476,7 +480,9 @@ public sealed class GatingReasonServiceTests DsseEnvelopeHash = null, SignatureRef = null, SourceDomain = "unknown", - SourceRef = "unknown" + SourceRef = "unknown", + ValidFrom = now, + CollectedAt = now }; // Assert - base trust only @@ -493,12 +499,16 @@ public sealed class GatingReasonServiceTests public void TriageFinding_RequiredFields_AreSet() { // Arrange + var now = DateTimeOffset.UtcNow; var finding = new TriageFinding { Id = Guid.NewGuid(), AssetLabel = "test-asset", Purl = "pkg:npm/test@1.0.0", - CveId = "CVE-2024-1234" + CveId = "CVE-2024-1234", + FirstSeenAt = now, + LastSeenAt = now, + UpdatedAt = now }; // Assert @@ -519,7 +529,8 @@ public sealed class GatingReasonServiceTests { Id = Guid.NewGuid(), PolicyId = "test-policy", - Action = action + Action = action, + AppliedAt = DateTimeOffset.UtcNow }; decision.Action.Should().Be(action); @@ -562,7 +573,8 @@ public sealed class GatingReasonServiceTests Id = Guid.NewGuid(), Reachable = TriageReachability.No, InputsHash = "sha256:inputs-hash", - SubgraphId = "sha256:subgraph" + SubgraphId = "sha256:subgraph", + ComputedAt = DateTimeOffset.UtcNow }; // Assert diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/LinksetResolverTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/LinksetResolverTests.cs index a141bcf7f..762d9f55f 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/LinksetResolverTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/LinksetResolverTests.cs @@ -113,7 +113,10 @@ public sealed class LinksetResolverTests FeatureFlags: Array.Empty(), Secrets: new SurfaceSecretsConfiguration("file", "tenant-a", "/etc/secrets", null, null, false), Tenant: "tenant-a", - Tls: new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())); + Tls: new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())) + { + CreatedAtUtc = DateTimeOffset.UtcNow + }; public IReadOnlyDictionary RawVariables { get; } = new Dictionary(StringComparer.Ordinal) { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScannerSurfaceSecretConfiguratorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScannerSurfaceSecretConfiguratorTests.cs index 1c30865dd..41df0e5c5 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScannerSurfaceSecretConfiguratorTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/ScannerSurfaceSecretConfiguratorTests.cs @@ -202,7 +202,10 @@ public sealed class ScannerSurfaceSecretConfiguratorTests Array.Empty(), new SurfaceSecretsConfiguration("inline", "tenant", null, null, null, true), "tenant", - new SurfaceTlsConfiguration(null, null, null)); + new SurfaceTlsConfiguration(null, null, null)) + { + CreatedAtUtc = DateTimeOffset.UtcNow + }; RawVariables = new Dictionary(StringComparer.OrdinalIgnoreCase); } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceCacheOptionsConfiguratorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceCacheOptionsConfiguratorTests.cs index 869a75a70..693e799be 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceCacheOptionsConfiguratorTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceCacheOptionsConfiguratorTests.cs @@ -26,7 +26,10 @@ public sealed class SurfaceCacheOptionsConfiguratorTests Array.Empty(), new SurfaceSecretsConfiguration("file", "tenant-b", "/etc/secrets", null, null, false), "tenant-b", - new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())); + new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())) + { + CreatedAtUtc = DateTimeOffset.UtcNow + }; var environment = new StubSurfaceEnvironment(settings); var configurator = new SurfaceCacheOptionsConfigurator(environment); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs index bdbc82443..b6a542cb1 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.WebService.Tests/SurfaceManifestStoreOptionsConfiguratorTests.cs @@ -28,7 +28,10 @@ public sealed class SurfaceManifestStoreOptionsConfiguratorTests Array.Empty(), new SurfaceSecretsConfiguration("file", "tenant-a", "/etc/secrets", null, null, false), "tenant-a", - new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())); + new SurfaceTlsConfiguration(null, null, new X509Certificate2Collection())) + { + CreatedAtUtc = DateTimeOffset.UtcNow + }; var environment = new StubSurfaceEnvironment(settings); var cacheOptions = Microsoft.Extensions.Options.Options.Create(new SurfaceCacheOptions { RootDirectory = cacheRoot.FullName });