diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f7bffe5c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Ensure analyzer fixture assets keep LF endings for deterministic hashes +src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/** text eol=lf diff --git a/.gitea/workflows/build-test-deploy.yml b/.gitea/workflows/build-test-deploy.yml index 90b850e0..ae972478 100644 --- a/.gitea/workflows/build-test-deploy.yml +++ b/.gitea/workflows/build-test-deploy.yml @@ -108,6 +108,36 @@ jobs: --logger "trx;LogFileName=stellaops-scanner-lang-tests.trx" \ --results-directory "$TEST_RESULTS_DIR" + - name: Run scanner analyzer performance benchmark + env: + PERF_OUTPUT_DIR: ${{ github.workspace }}/artifacts/perf/scanner-analyzers + PERF_ENVIRONMENT: ${{ github.event_name == 'pull_request' && 'preview' || 'staging' }} + run: | + set -euo pipefail + mkdir -p "$PERF_OUTPUT_DIR" + CAPTURED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + + dotnet run \ + --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj \ + --configuration $BUILD_CONFIGURATION \ + -- \ + --repo-root . \ + --baseline bench/Scanner.Analyzers/baseline.csv \ + --out "$PERF_OUTPUT_DIR/latest.csv" \ + --json "$PERF_OUTPUT_DIR/report.json" \ + --prom "$PERF_OUTPUT_DIR/metrics.prom" \ + --commit "${GITHUB_SHA}" \ + --environment "$PERF_ENVIRONMENT" \ + --captured-at "$CAPTURED_AT" + + - name: Upload scanner analyzer benchmark artifacts + uses: actions/upload-artifact@v4 + with: + name: scanner-analyzers-benchmark + path: artifacts/perf/scanner-analyzers + if-no-files-found: error + retention-days: 14 + - name: Publish BuildX SBOM generator run: | dotnet publish src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj \ diff --git a/EXECPLAN.md b/EXECPLAN.md index d72393e5..3c1e0c11 100644 --- a/EXECPLAN.md +++ b/EXECPLAN.md @@ -4,16 +4,17 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster ## Wave Instructions ### Wave 0 - Team Attestor Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Attestor/TASKS.md`. Focus on ATTESTOR-API-11-201 (TODO), ATTESTOR-VERIFY-11-202 (TODO), ATTESTOR-OBS-11-203 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team Authority Core & Security Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTH-DPOP-11-001 (DONE 2025-10-20), AUTH-MTLS-11-002 (DOING 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Authority Core & Security Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTH-DPOP-11-001 (DONE 2025-10-20), AUTH-MTLS-11-002 (DONE 2025-10-23). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Authority Core & Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTHSTORAGE-MONGO-08-001 (DONE 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team DevEx/CLI: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on EXCITITOR-CLI-01-002 (TODO), CLI-RUNTIME-13-005 (TODO). Confirm prerequisites (external: EXCITITOR-CLI-01-001, EXCITITOR-EXPORT-01-001) before starting and report status in module TASKS.md. - Team DevOps Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SEC-10-301 (DONE 2025-10-20); Wave 0A prerequisites reconfirmed so remediation work may proceed. Keep module TASKS.md/Sprints in sync as patches land. -- Team Diff Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Diff/TASKS.md`. Focus on SCANNER-DIFF-10-501 (TODO), SCANNER-DIFF-10-502 (TODO), SCANNER-DIFF-10-503 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Diff Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Diff/TASKS.md`. SCANNER-DIFF-10-501/502/503 all closed on 2025-10-19; keep determinism fixtures green and sync downstream consumers as Emit/Diff integration tickets arise. +- Team Scanner Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Storage/TASKS.md`. Focus on SCANNER-STORAGE-11-401 (DONE 2025-10-23) to migrate MinIO integrations to RustFS; ensure prerequisites (SCANNER-STORAGE-09-302) stay satisfied before execution and record status in module TASKS.md. - Team Docs Guild, Plugin Team: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `docs/TASKS.md`. Focus on DOC4.AUTH-PDG (REVIEW). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Docs/CLI: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on EXCITITOR-CLI-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-CLI-01-001) before starting and report status in module TASKS.md. -- Team Emit Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Emit/TASKS.md`. Focus on SCANNER-EMIT-10-601 (TODO), SCANNER-EMIT-10-602 (TODO), SCANNER-EMIT-10-603 (TODO), SCANNER-EMIT-10-604 (TODO), SCANNER-EMIT-10-605 (TODO), SCANNER-EMIT-10-606 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team EntryTrace Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.EntryTrace/TASKS.md`. Focus on SCANNER-ENTRYTRACE-10-401 (TODO), SCANNER-ENTRYTRACE-10-402 (TODO), SCANNER-ENTRYTRACE-10-403 (TODO), SCANNER-ENTRYTRACE-10-404 (TODO), SCANNER-ENTRYTRACE-10-405 (TODO), SCANNER-ENTRYTRACE-10-406 (TODO), SCANNER-ENTRYTRACE-10-407 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. -- Team Language Analyzer Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md`, `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-301 (TODO) and the upcoming Python/Go/.NET/Rust analyzers (10-303..306). Node sprint items 10-302/307/308/309 are DONE (latest 2025-10-21); shift coordination to remaining ecosystem analyzers and track follow-up work via module TASKS.md. +- Team Emit Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Emit/TASKS.md`. Sprint 10 composition milestones (10-601..10-606) wrapped 2025-10-22 and SCANNER-EMIT-10-607 completed alongside; remaining watch item is SCANNER-EMIT-17-701 (Wave 1) with build-id enrichment. +- Team EntryTrace Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.EntryTrace/TASKS.md`. SCANNER-ENTRYTRACE-10-401..407 landed 2025-10-19; continue monitoring determinism harness outputs and raise follow-ups if new interpreter cases appear. +- Team Language Analyzer Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md`, `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md`. Java, shared helpers, determinism harness, and the Sprint 10 analyzers (10-301..10-309) are DONE (latest 2025-10-22); keep fixture refresh notes current and pivot to Wave 1 benchmarking/packaging follow-ups. - Team Notify Models Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.Models/TASKS.md`. Focus on NOTIFY-MODELS-15-101 (TODO), NOTIFY-MODELS-15-102 (TODO), NOTIFY-MODELS-15-103 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Notify Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.Storage.Mongo/TASKS.md`. Focus on NOTIFY-STORAGE-15-201 (TODO), NOTIFY-STORAGE-15-202 (TODO), NOTIFY-STORAGE-15-203 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. - Team Notify WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-101 (TODO), NOTIFY-WEB-15-102 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. @@ -56,8 +57,8 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team DevEx/CLI, QA Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-RUNTIME-13-009 (TODO). Confirm prerequisites (internal: CLI-RUNTIME-13-005 (Wave 0)) before starting and report status in module TASKS.md. - Team DevOps Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-REL-14-001 (TODO). Confirm prerequisites (internal: ATTESTOR-API-11-201 (Wave 0), SIGNER-API-11-101 (Wave 0)) before starting and report status in module TASKS.md. - Team DevOps Guild, Scanner WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SCANNER-09-204 (TODO). Confirm prerequisites (internal: SCANNER-EVENTS-15-201 (Wave 0)) before starting and report status in module TASKS.md. -- Team Emit Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Emit/TASKS.md`. Focus on SCANNER-EMIT-10-607 (TODO), SCANNER-EMIT-17-701 (TODO). Confirm prerequisites (internal: POLICY-CORE-09-005 (Wave 0), SCANNER-EMIT-10-602 (Wave 0), SCANNER-EMIT-10-604 (Wave 0)) before starting and report status in module TASKS.md. -- Team Language Analyzer Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-303 (DONE 2025-10-21), SCANNER-ANALYZERS-LANG-10-304 (DOING 2025-10-22), SCANNER-ANALYZERS-LANG-10-305 (DOING 2025-10-22), SCANNER-ANALYZERS-LANG-10-306 (TODO). Node stream (tasks 10-302/309) closed on 2025-10-21; verify prereqs SCANNER-ANALYZERS-LANG-10-301/307 remain satisfied before pivoting to the remaining language analyzers. +- Team Emit Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Emit/TASKS.md`. SCANNER-EMIT-10-607 shipped 2025-10-22; remaining focus is SCANNER-EMIT-17-701 (build-id enrichment). Confirm prerequisites (internal: POLICY-CORE-09-005 (Wave 0), SCANNER-EMIT-10-602 (Wave 0), SCANNER-EMIT-10-604 (Wave 0)) before starting and report status in module TASKS.md. +- Team Language Analyzer Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md`. Sprint 10 language analyzers (10-303..10-306) wrapped by 2025-10-22; shift to Wave 1 benchmarking/packaging follow-ups (10-308+/309 variants) and ensure shared helpers stay stable. Node stream (tasks 10-302/309) closed on 2025-10-21; verify prereqs SCANNER-ANALYZERS-LANG-10-301/307 remain satisfied before new work. - Team Licensing Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `ops/licensing/TASKS.md`. Focus on DEVOPS-LIC-14-004 (TODO). Confirm prerequisites (internal: AUTH-MTLS-11-002 (Wave 0)) before starting and report status in module TASKS.md. - Team Notify Engine Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-301 (TODO). Confirm prerequisites (internal: NOTIFY-MODELS-15-101 (Wave 0)) before starting and report status in module TASKS.md. - Team Notify Queue Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.Queue/TASKS.md`. Focus on NOTIFY-QUEUE-15-401 (TODO). Confirm prerequisites (internal: NOTIFY-MODELS-15-101 (Wave 0)) before starting and report status in module TASKS.md. @@ -68,7 +69,7 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Scheduler Storage Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md`. Focus on SCHED-STORAGE-16-203 (TODO), SCHED-STORAGE-16-202 (TODO). Confirm prerequisites (internal: SCHED-STORAGE-16-201 (Wave 0)) before starting and report status in module TASKS.md. - Team Scheduler WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-104 (TODO), SCHED-WEB-16-102 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0), SCHED-STORAGE-16-201 (Wave 0), SCHED-WEB-16-101 (Wave 0)) before starting and report status in module TASKS.md. - Team Scheduler Worker Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-201 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0)) before starting and report status in module TASKS.md. -- Team TBD: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-305A (DONE 2025-10-22), SCANNER-ANALYZERS-LANG-10-304A (DONE 2025-10-22), SCANNER-ANALYZERS-LANG-10-303A (DONE 2025-10-21), SCANNER-ANALYZERS-LANG-10-306A (TODO); Node add-ons 10-307N/10-308N/10-309N now DONE with restart-time packaging verified 2025-10-21. Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-302C (Wave 0), SCANNER-ANALYZERS-LANG-10-307 (Wave 0)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. SCANNER-ANALYZERS-LANG-10-305A/304A/303A/306A all closed by 2025-10-22; use this slot to review cross-language fixture hygiene and prep Wave 1 benchmarking tickets. Node add-ons 10-307N/10-308N/10-309N remain DONE with restart-time packaging verified 2025-10-21. Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-302C (Wave 0), SCANNER-ANALYZERS-LANG-10-307 (Wave 0)) before starting any new follow-ups and report status in module TASKS.md. - Team Team Excititor Connectors – MSRC: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-MS-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-MS-01-002 (Wave 0); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. - Team Team Excititor Connectors – Oracle: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-ORACLE-01-002 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-ORACLE-01-001 (Wave 0); external: EXCITITOR-STORAGE-01-003) before starting and report status in module TASKS.md. - Team Team Excititor Connectors – SUSE: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md`. Focus on EXCITITOR-CONN-SUSE-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-SUSE-01-002 (Wave 0); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. @@ -82,19 +83,19 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Bench Guild, Notify Team: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `bench/TASKS.md`. Focus on BENCH-NOTIFY-15-001 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-301 (Wave 1)) before starting and report status in module TASKS.md. - Team Bench Guild, Scheduler Team: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `bench/TASKS.md`. Focus on BENCH-IMPACT-16-001 (TODO). Confirm prerequisites (internal: SCHED-IMPACT-16-301 (Wave 1)) before starting and report status in module TASKS.md. - Team Deployment Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/deployment/TASKS.md`. Focus on DEVOPS-OPS-14-003 (TODO). Confirm prerequisites (internal: DEVOPS-REL-14-001 (Wave 1)) before starting and report status in module TASKS.md. -- Team DevOps Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-MIRROR-08-001 (DONE 2025-10-19), DEVOPS-PERF-10-002 (TODO), DEVOPS-REL-17-002 (TODO), and DEVOPS-NUGET-13-001 (TODO). Confirm prerequisites (internal: BENCH-SCANNER-10-002 (Wave 1), DEVOPS-REL-14-001 (Wave 1), SCANNER-EMIT-17-701 (Wave 1)) before starting and report status in module TASKS.md. +- Team DevOps Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-MIRROR-08-001 (DONE 2025-10-19), DEVOPS-PERF-10-002 (TODO), DEVOPS-REL-14-004 (TODO), DEVOPS-REL-17-002 (TODO), and DEVOPS-NUGET-13-001 (TODO). Confirm prerequisites (internal: BENCH-SCANNER-10-002 (Wave 1), DEVOPS-REL-14-001 (Wave 1), SCANNER-EMIT-17-701 (Wave 1)) before starting and report status in module TASKS.md. - Team DevOps Guild, Notify Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SCANNER-09-205 (TODO). Confirm prerequisites (internal: DEVOPS-SCANNER-09-204 (Wave 1)) before starting and report status in module TASKS.md. - Team Notify Engine Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-302 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-301 (Wave 1)) before starting and report status in module TASKS.md. - Team Notify Queue Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.Queue/TASKS.md`. Focus on NOTIFY-QUEUE-15-403 (TODO), NOTIFY-QUEUE-15-402 (TODO). Confirm prerequisites (internal: NOTIFY-QUEUE-15-401 (Wave 1)) before starting and report status in module TASKS.md. - Team Notify WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-104 (TODO). Confirm prerequisites (internal: NOTIFY-QUEUE-15-401 (Wave 1), NOTIFY-STORAGE-15-201 (Wave 0)) before starting and report status in module TASKS.md. - Team Notify Worker Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.Worker/TASKS.md`. Focus on NOTIFY-WORKER-15-201 (TODO), NOTIFY-WORKER-15-202 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-301 (Wave 1), NOTIFY-QUEUE-15-401 (Wave 1)) before starting and report status in module TASKS.md. -- Team Offline Kit Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/offline-kit/TASKS.md`. Focus on DEVOPS-OFFLINE-14-002 (TODO) and DEVOPS-OFFLINE-18-003 (TODO). Confirm prerequisites (internal: DEVOPS-REL-14-001 (Wave 1)) before starting and report status in module TASKS.md. +- Team Offline Kit Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/offline-kit/TASKS.md`. Focus on DEVOPS-OFFLINE-14-002 (TODO), DEVOPS-OFFLINE-18-003 (TODO), and DEVOPS-OFFLINE-18-005 (TODO). Confirm prerequisites (internal: DEVOPS-REL-14-001 (Wave 1), DEVOPS-REL-14-004 (Wave 2)) before starting and report status in module TASKS.md. - Team Samples Guild, Policy Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `samples/TASKS.md`. Focus on SAMPLES-13-004 (TODO). Confirm prerequisites (internal: POLICY-CORE-09-006 (Wave 0), UI-POLICY-13-007 (Wave 1)) before starting and report status in module TASKS.md. - Team Scanner WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-12-302 (TODO). Confirm prerequisites (internal: SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-CORE-12-201 (Wave 0)) before starting and report status in module TASKS.md. - Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-303 (TODO), SCHED-IMPACT-16-302 (TODO). Confirm prerequisites (internal: SCHED-IMPACT-16-301 (Wave 1)) before starting and report status in module TASKS.md. - Team Scheduler WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-103 (TODO). Confirm prerequisites (internal: SCHED-WEB-16-102 (Wave 1)) before starting and report status in module TASKS.md. - Team Scheduler Worker Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-202 (TODO), SCHED-WORKER-16-205 (TODO). Confirm prerequisites (internal: SCHED-IMPACT-16-301 (Wave 1), SCHED-WORKER-16-201 (Wave 1)) before starting and report status in module TASKS.md. -- Team TBD: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-305B (DONE 2025-10-22), SCANNER-ANALYZERS-LANG-10-304B (DONE 2025-10-22), SCANNER-ANALYZERS-LANG-10-303B (DONE 2025-10-21), SCANNER-ANALYZERS-LANG-10-306B (TODO); Node packaging milestone 10-308N closed 2025-10-21. Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303A (Wave 1), SCANNER-ANALYZERS-LANG-10-304A (Wave 1), SCANNER-ANALYZERS-LANG-10-305A (Wave 1), SCANNER-ANALYZERS-LANG-10-306A (Wave 1), SCANNER-ANALYZERS-LANG-10-307N (Wave 1)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. SCANNER-ANALYZERS-LANG-10-305B/304B/303B/306B wrapped on 2025-10-22; next focus moves to `10-307*` shared helper integration and Wave 2 benchmark polish. Node packaging milestone 10-308N closed 2025-10-21. Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303A (Wave 1), SCANNER-ANALYZERS-LANG-10-304A (Wave 1), SCANNER-ANALYZERS-LANG-10-305A (Wave 1), SCANNER-ANALYZERS-LANG-10-306A (Wave 1), SCANNER-ANALYZERS-LANG-10-307N (Wave 1)) before starting new work and report status in module TASKS.md. - Team Team Excititor Connectors – Oracle: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-ORACLE-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-ORACLE-01-002 (Wave 1); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. - Team Team Excititor Export: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Excititor.Export/TASKS.md`. Focus on EXCITITOR-EXPORT-01-007 (DONE 2025-10-21). Confirm prerequisites (internal: EXCITITOR-EXPORT-01-006 (Wave 1)) before starting and report status in module TASKS.md. - Team Zastava Observer Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Zastava.Observer/TASKS.md`. Focus on ZASTAVA-OBS-12-002 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-12-001 (Wave 1)) before starting and report status in module TASKS.md. @@ -106,7 +107,7 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Notify Engine Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-303 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-302 (Wave 2)) before starting and report status in module TASKS.md. - Team Notify Worker Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Notify.Worker/TASKS.md`. Focus on NOTIFY-WORKER-15-203 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-302 (Wave 2)) before starting and report status in module TASKS.md. - Team Scheduler Worker Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-203 (TODO). Confirm prerequisites (internal: SCHED-WORKER-16-202 (Wave 2)) before starting and report status in module TASKS.md. -- Team TBD: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-305C (DONE 2025-10-22), SCANNER-ANALYZERS-LANG-10-304C (TODO), SCANNER-ANALYZERS-LANG-10-309N (TODO), SCANNER-ANALYZERS-LANG-10-303C (DONE 2025-10-21), SCANNER-ANALYZERS-LANG-10-306C (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303B (Wave 2), SCANNER-ANALYZERS-LANG-10-304B (Wave 2), SCANNER-ANALYZERS-LANG-10-305B (Wave 2), SCANNER-ANALYZERS-LANG-10-306B (Wave 2), SCANNER-ANALYZERS-LANG-10-308N (Wave 2)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. SCANNER-ANALYZERS-LANG-10-305C/304C/309N/303C/306C are all DONE (latest 2025-10-22); remaining Wave 3 attention shifts to 10-307* helper consolidation and subsequent benchmarking tickets. Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303B (Wave 2), SCANNER-ANALYZERS-LANG-10-304B (Wave 2), SCANNER-ANALYZERS-LANG-10-305B (Wave 2), SCANNER-ANALYZERS-LANG-10-306B (Wave 2), SCANNER-ANALYZERS-LANG-10-308N (Wave 2)) before scheduling new work and report status in module TASKS.md. - Team Zastava Observer Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Zastava.Observer/TASKS.md`. Focus on ZASTAVA-OBS-12-003 (TODO), ZASTAVA-OBS-12-004 (TODO), ZASTAVA-OBS-17-005 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-12-002 (Wave 2)) before starting and report status in module TASKS.md. ### Wave 4 @@ -118,17 +119,17 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Team Notify Worker Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Notify.Worker/TASKS.md`. Focus on NOTIFY-WORKER-15-204 (TODO). Confirm prerequisites (internal: NOTIFY-WORKER-15-203 (Wave 3)) before starting and report status in module TASKS.md. - Team Policy Guild, Scanner WebService Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Policy/TASKS.md`. Focus on POLICY-RUNTIME-17-201 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-17-005 (Wave 3)) before starting and report status in module TASKS.md. - Team Scheduler Worker Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-204 (TODO). Confirm prerequisites (internal: SCHED-WORKER-16-203 (Wave 3)) before starting and report status in module TASKS.md. -- Team TBD: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-307D (DONE 2025-10-22), SCANNER-ANALYZERS-LANG-10-307G (TODO), SCANNER-ANALYZERS-LANG-10-307P (TODO), SCANNER-ANALYZERS-LANG-10-307R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303C (Wave 3), SCANNER-ANALYZERS-LANG-10-304C (Wave 3), SCANNER-ANALYZERS-LANG-10-305C (Wave 3), SCANNER-ANALYZERS-LANG-10-306C (Wave 3)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. SCANNER-ANALYZERS-LANG-10-307D/G/P are DONE (latest 2025-10-23); remaining focus is SCANNER-ANALYZERS-LANG-10-307R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303C (Wave 3), SCANNER-ANALYZERS-LANG-10-304C (Wave 3), SCANNER-ANALYZERS-LANG-10-305C (Wave 3), SCANNER-ANALYZERS-LANG-10-306C (Wave 3)) before progressing and report status in module TASKS.md. ### Wave 5 - Team Excititor Connectors – Stella: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md`. Focus on EXCITITOR-CONN-STELLA-07-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-STELLA-07-002 (Wave 4)) before starting and report status in module TASKS.md. - Team Notify Connectors Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-502 (DONE), NOTIFY-CONN-TEAMS-15-602 (DONE), NOTIFY-CONN-EMAIL-15-702 (BLOCKED 2025-10-20), NOTIFY-CONN-WEBHOOK-15-802 (BLOCKED 2025-10-20). Confirm prerequisites (internal: NOTIFY-CONN-EMAIL-15-701 (Wave 4), NOTIFY-CONN-SLACK-15-501 (Wave 4), NOTIFY-CONN-TEAMS-15-601 (Wave 4), NOTIFY-CONN-WEBHOOK-15-801 (Wave 4)) before starting and report status in module TASKS.md. - Team Scanner WebService Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-17-401 (TODO). Confirm prerequisites (internal: POLICY-RUNTIME-17-201 (Wave 4), SCANNER-EMIT-17-701 (Wave 1), SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-OBS-17-005 (Wave 3)) before starting and report status in module TASKS.md. -- Team TBD: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-308D (DONE 2025-10-23), SCANNER-ANALYZERS-LANG-10-308G (TODO), SCANNER-ANALYZERS-LANG-10-308P (TODO), SCANNER-ANALYZERS-LANG-10-308R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-307D (Wave 4), SCANNER-ANALYZERS-LANG-10-307G (Wave 4), SCANNER-ANALYZERS-LANG-10-307P (Wave 4), SCANNER-ANALYZERS-LANG-10-307R (Wave 4)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. SCANNER-ANALYZERS-LANG-10-308D/G/P completed (2025-10-23/2025-10-22/2025-10-23); pending items are SCANNER-ANALYZERS-LANG-10-308R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-307D (Wave 4), SCANNER-ANALYZERS-LANG-10-307G (Wave 4), SCANNER-ANALYZERS-LANG-10-307P (Wave 4), SCANNER-ANALYZERS-LANG-10-307R (Wave 4)) before starting and report status in module TASKS.md. ### Wave 6 - Team Notify Connectors Guild: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-503 (DONE), NOTIFY-CONN-TEAMS-15-603 (DONE), NOTIFY-CONN-EMAIL-15-703 (DONE), NOTIFY-CONN-WEBHOOK-15-803 (DONE). Confirm packaging outputs remain deterministic while upstream implementation tasks (15-702/802) stay blocked. -- Team TBD: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-309D (DONE 2025-10-23), SCANNER-ANALYZERS-LANG-10-309G (TODO), SCANNER-ANALYZERS-LANG-10-309P (TODO), SCANNER-ANALYZERS-LANG-10-309R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-308D (Wave 5), SCANNER-ANALYZERS-LANG-10-308G (Wave 5), SCANNER-ANALYZERS-LANG-10-308P (Wave 5), SCANNER-ANALYZERS-LANG-10-308R (Wave 5)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. SCANNER-ANALYZERS-LANG-10-309D/G/P completed (2025-10-23/2025-10-22/2025-10-23); remaining item is SCANNER-ANALYZERS-LANG-10-309R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-308D (Wave 5), SCANNER-ANALYZERS-LANG-10-308G (Wave 5), SCANNER-ANALYZERS-LANG-10-308P (Wave 5), SCANNER-ANALYZERS-LANG-10-308R (Wave 5)) before starting and report status in module TASKS.md. ### Wave 7 - Team Team Core Engine & Storage Analytics: read EXECPLAN.md Wave 7 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-001 (DONE 2025-10-19). Confirm prerequisites (internal: FEEDSTORAGE-DATA-07-001 (Wave 10)) before starting and report status in module TASKS.md. @@ -317,63 +318,63 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - **Sprint 10** · Scanner Analyzers & SBOM - Team: Diff Guild - Path: `src/StellaOps.Scanner.Diff/TASKS.md` - 1. [TODO] SCANNER-DIFF-10-501 — Build component differ tracking add/remove/version changes with deterministic ordering. + 1. [DONE 2025-10-19] SCANNER-DIFF-10-501 — Build component differ tracking add/remove/version changes with deterministic ordering. • Prereqs: — - • Current: TODO - 2. [TODO] SCANNER-DIFF-10-502 — Attribute diffs to introducing/removing layers including provenance evidence. + • Current: DONE — Diff engine produces deterministic add/remove/version deltas; regression suite covers warm/cold path parity. + 2. [DONE 2025-10-19] SCANNER-DIFF-10-502 — Attribute diffs to introducing/removing layers including provenance evidence. • Prereqs: — - • Current: TODO - 3. [TODO] SCANNER-DIFF-10-503 — Produce JSON diff output for inventory vs usage views aligned with API contract. + • Current: DONE — Layer attribution recorded on every change; fixtures assert provenance integrity. + 3. [DONE 2025-10-19] SCANNER-DIFF-10-503 — Produce JSON diff output for inventory vs usage views aligned with API contract. • Prereqs: — - • Current: TODO + • Current: DONE — JSON serializer emits stable ordering; golden outputs locked in tests. - Team: Emit Guild - Path: `src/StellaOps.Scanner.Emit/TASKS.md` - 1. [TODO] SCANNER-EMIT-10-601 — Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments. + 1. [DONE 2025-10-22] SCANNER-EMIT-10-601 — Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments. • Prereqs: — - • Current: TODO - 2. [TODO] SCANNER-EMIT-10-602 — Compose usage SBOM leveraging EntryTrace to flag actual usage. + • Current: DONE — Inventory builder validated against CycloneDX schema; deterministic fixtures added. + 2. [DONE 2025-10-22] SCANNER-EMIT-10-602 — Compose usage SBOM leveraging EntryTrace to flag actual usage. • Prereqs: — - • Current: TODO - 3. [TODO] SCANNER-EMIT-10-603 — Generate BOM index sidecar (purl table + roaring bitmap + usage flag). + • Current: DONE — Usage view toggles wired; tests confirm subset alignment with EntryTrace signals. + 3. [DONE 2025-10-22] SCANNER-EMIT-10-603 — Generate BOM index sidecar (purl table + roaring bitmap + usage flag). • Prereqs: — - • Current: TODO - 4. [TODO] SCANNER-EMIT-10-604 — Package artifacts for export + attestation with deterministic manifests. + • Current: DONE — BOM Index format published with roaring bitmap helpers; golden fixtures locked. + 4. [DONE 2025-10-22] SCANNER-EMIT-10-604 — Package artifacts for export + attestation with deterministic manifests. • Prereqs: — - • Current: TODO - 5. [TODO] SCANNER-EMIT-10-605 — Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). + • Current: DONE — Export packaging deterministic; integration test with storage succeeds. + 5. [DONE 2025-10-22] SCANNER-EMIT-10-605 — Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). • Prereqs: — - • Current: TODO - 6. [TODO] SCANNER-EMIT-10-606 — Usage view bit flags integrated with EntryTrace. + • Current: DONE — `bom-index@1` schema + fixtures published; Scheduler notes updated. + 6. [DONE 2025-10-22] SCANNER-EMIT-10-606 — Usage view bit flags integrated with EntryTrace. • Prereqs: — - • Current: TODO + • Current: DONE — EntryTrace usage bits round-trip in BOM Index; regression harness verified. - Team: EntryTrace Guild - Path: `src/StellaOps.Scanner.EntryTrace/TASKS.md` - 1. [TODO] SCANNER-ENTRYTRACE-10-401 — POSIX shell AST parser with deterministic output. + 1. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-401 — POSIX shell AST parser with deterministic output. • Prereqs: — - • Current: TODO - 2. [TODO] SCANNER-ENTRYTRACE-10-402 — Command resolution across layered rootfs with evidence attribution. + • Current: DONE — Parser emits stable AST; determinism tests captured. + 2. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-402 — Command resolution across layered rootfs with evidence attribution. • Prereqs: — - • Current: TODO - 3. [TODO] SCANNER-ENTRYTRACE-10-403 — Interpreter tracing for shell wrappers to Python/Node/Java launchers. + • Current: DONE — Resolver walks layered PATH with provenance evidence; fixtures validate. + 3. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-403 — Interpreter tracing for shell wrappers to Python/Node/Java launchers. • Prereqs: — - • Current: TODO - 4. [TODO] SCANNER-ENTRYTRACE-10-404 — Python entry analyzer (venv shebang, module invocation, usage flag). + • Current: DONE — Interpreter tracer resolves Python/Node/Java hand-offs; golden graphs updated. + 4. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-404 — Python entry analyzer (venv shebang, module invocation, usage flag). • Prereqs: — - • Current: TODO - 5. [TODO] SCANNER-ENTRYTRACE-10-405 — Node/Java launcher analyzer capturing script/jar targets. + • Current: DONE — Python analyzer surfaces venv/module details; usage flag propagated. + 5. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-405 — Node/Java launcher analyzer capturing script/jar targets. • Prereqs: — - • Current: TODO - 6. [TODO] SCANNER-ENTRYTRACE-10-406 — Explainability + diagnostics for unresolved constructs with metrics. + • Current: DONE — Node/Java launchers traced end-to-end; evidence attached for each hop. + 6. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-406 — Explainability + diagnostics for unresolved constructs with metrics. • Prereqs: — - • Current: TODO - 7. [TODO] SCANNER-ENTRYTRACE-10-407 — Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). + • Current: DONE — Diagnostics enumerated, metrics emitted via `EntryTraceMetrics`. + 7. [DONE 2025-10-19] SCANNER-ENTRYTRACE-10-407 — Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). • Prereqs: — - • Current: TODO + • Current: DONE — Plug-in manifests under `plugins/scanner/entrytrace`; restart-only guard documented. - Team: Language Analyzer Guild - Path: `src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-302..309 — Detailed per-language sprint plan (Node, Python, Go, .NET, Rust) with gates and benchmarks. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-302..309 — Detailed per-language sprint plan (Node, Python, Go, .NET, Rust) with gates and benchmarks. • Prereqs: — - • Current: TODO + • Current: DONE — Implementation plan captured per language with progress notes through 2025-10-22. - Path: `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md` 1. [DONE 2025-10-19] SCANNER-ANALYZERS-LANG-10-301 — Java analyzer emitting `pkg:maven` with provenance. • Prereqs: — @@ -389,18 +390,23 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Path: `src/StellaOps.Attestor/TASKS.md` 1. [TODO] ATTESTOR-API-11-201 — `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. • Prereqs: — - • Current: TODO + • Current: DOING (2025-10-23) — RustFS migration underway. 2. [TODO] ATTESTOR-VERIFY-11-202 — `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. • Prereqs: — • Current: TODO 3. [TODO] ATTESTOR-OBS-11-203 — Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. • Prereqs: — • Current: TODO + - Team: Scanner Storage Guild + - Path: `src/StellaOps.Scanner.Storage/TASKS.md` + 1. [DONE 2025-10-23] SCANNER-STORAGE-11-401 — Migrate scanner artifact storage from MinIO to RustFS, including driver, configuration, and migration tooling. + • Prereqs: SCANNER-STORAGE-09-302 (Wave 0) + • Current: DONE — RustFS driver, deployment manifests, migration tool, and test coverage shipped. - Team: Authority Core & Security Guild - Path: `src/StellaOps.Authority/TASKS.md` - 2. [DOING] AUTH-MTLS-11-002 — Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. + 2. [DONE 2025-10-23] AUTH-MTLS-11-002 — Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. • Prereqs: — - • Current: DOING (2025-10-19) + • Current: DONE — mTLS audience enforcement + certificate binding validation shipped; docs/tests updated. - **Sprint 12** · Runtime Guardrails - Team: Zastava Core Guild @@ -552,32 +558,32 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) • Current: DONE — Python analyzer ingests METADATA/WHEEL/entry_points with deterministic ordering and UTF-8 normalization. Fixtures updated (`simple-venv`). - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-306A — Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-306A — Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) - • Current: TODO + • Current: DONE — Cargo metadata walker emits `pkg:cargo` components with provenance and deterministic fixtures. - **Sprint 10** · Scanner Analyzers & SBOM - Team: Emit Guild - Path: `src/StellaOps.Scanner.Emit/TASKS.md` - 1. [TODO] SCANNER-EMIT-10-607 — Embed scoring inputs, confidence band, and `quietedBy` provenance into CycloneDX 1.6 and DSSE predicates; verify deterministic serialization. + 1. [DONE 2025-10-22] SCANNER-EMIT-10-607 — Embed scoring inputs, confidence band, and `quietedBy` provenance into CycloneDX 1.6 and DSSE predicates; verify deterministic serialization. • Prereqs: SCANNER-EMIT-10-604 (Wave 0), POLICY-CORE-09-005 (Wave 0) - • Current: TODO + • Current: DONE — SBOM/attestation fixtures include scoring metadata and serialize deterministically. - Team: Language Analyzer Guild - Path: `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md` 1. [DONE 2025-10-21] SCANNER-ANALYZERS-LANG-10-309 — Package language analyzers as restart-time plug-ins (manifest + host registration). • Prereqs: SCANNER-ANALYZERS-LANG-10-301 (Wave 0) • Current: DONE — Manifest published under `plugins/scanner/analyzers/lang/`, Worker loader wired, integration tests updated. - 2. [TODO] SCANNER-ANALYZERS-LANG-10-306 — Rust analyzer detecting crate provenance or falling back to `bin:{sha256}`. + 2. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-306 — Rust analyzer detecting crate provenance or falling back to `bin:{sha256}`. • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) - • Current: TODO + • Current: DONE — Rust analyzer emits cargo components with provenance and deterministic fallbacks. 3. [DONE 2025-10-21] SCANNER-ANALYZERS-LANG-10-302 — Node analyzer resolving workspaces/symlinks into `pkg:npm` identities. • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) • Current: DONE — Workspace/symlink coverage validated via determinism fixtures; metrics + lifecycle script evidence landed. - 4. [DOING 2025-10-22] SCANNER-ANALYZERS-LANG-10-304 — Go analyzer leveraging buildinfo for `pkg:golang` components. + 4. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-304 — Go analyzer leveraging buildinfo for `pkg:golang` components. • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) - • Current: TODO - 5. [DOING 2025-10-22] SCANNER-ANALYZERS-LANG-10-305 — .NET analyzer parsing `*.deps.json`, assembly metadata, and RID variants. + • Current: DONE — Buildinfo decoder + DWARF fallbacks captured; fixtures and benchmarks green. + 5. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-305 — .NET analyzer parsing `*.deps.json`, assembly metadata, and RID variants. • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) - • Current: DOING — Implementing initial deps/runtimeconfig parsing for RID-aware components. + • Current: DONE — RID-aware deps/runtimeconfig parser emits deterministic NuGet components; tests landed. 6. [DONE 2025-10-21] SCANNER-ANALYZERS-LANG-10-303 — Python analyzer consuming `*.dist-info` metadata and RECORD hashes. • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) • Current: DONE — Dist-info parser, RECORD verifier, editable install metadata, and entrypoint usage hints shipped with deterministic fixture/tests. @@ -737,21 +743,26 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Prereqs: SCANNER-ANALYZERS-LANG-10-303A (Wave 1) • Current: DONE — Streaming SHA-256 verification with deterministic mismatch evidence; unsupported algorithms tracked; fixtures validated. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-306B — Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-306B — Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. • Prereqs: SCANNER-ANALYZERS-LANG-10-306A (Wave 1) - • Current: TODO + • Current: DONE — Heuristic classifier flags stripped binaries, regression tests guard false positives. - **Sprint 10** · DevOps Perf - Team: DevOps Guild - Path: `ops/devops/TASKS.md` - 1. [TODO] DEVOPS-PERF-10-002 — Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. + 1. [DONE (2025-10-23)] DEVOPS-PERF-10-002 — Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. • Prereqs: BENCH-SCANNER-10-002 (Wave 1) - • Current: TODO + • Current: DONE (2025-10-23) - **Sprint 10** · Samples - Team: Samples Guild, Policy Guild - Path: `samples/TASKS.md` - 1. [TODO] SAMPLES-13-004 — Add policy preview/report fixtures showing confidence bands and unknown-age tags. + 1. [DONE (2025-10-23)] SAMPLES-13-004 — Add policy preview/report fixtures showing confidence bands and unknown-age tags. • Prereqs: POLICY-CORE-09-006 (Wave 0), UI-POLICY-13-007 (Wave 1) - • Current: TODO + • Current: DONE (2025-10-23) + - Team: UI Guild + - Path: `src/StellaOps.Web/TASKS.md` + 1. [DONE (2025-10-23)] WEB-POLICY-FIXTURES-10-001 — Wire policy preview/report doc fixtures into UI harness (test utility or Storybook substitute) with type bindings and validation guard. + • Prereqs: SAMPLES-13-004 (Wave 0) + • Current: DONE (2025-10-23) - **Sprint 12** · Runtime Guardrails - Team: Scanner WebService Guild - Path: `src/StellaOps.Scanner.WebService/TASKS.md` @@ -854,11 +865,11 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-305C — Handle self-contained apps and native assets; merge with EntryTrace usage hints. • Prereqs: SCANNER-ANALYZERS-LANG-10-305A (Wave 1) - • Current: TODO + • Current: DONE — Self-contained fixtures emit components with RID flags; EntryTrace usage hints preserved. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-304C — Fallback heuristics for stripped binaries with deterministic `bin:{sha256}` labeling and quiet provenance. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-304C — Fallback heuristics for stripped binaries with deterministic `bin:{sha256}` labeling and quiet provenance. • Prereqs: SCANNER-ANALYZERS-LANG-10-304B (Wave 2) - • Current: TODO + • Current: DONE — `bin:{sha256}` fallback + quiet provenance docs shipped with determinism fixtures. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md` 1. [DONE 2025-10-21] SCANNER-ANALYZERS-LANG-10-309N — Package Node analyzer as restart-time plug-in (manifest, DI registration, Offline Kit notes). • Prereqs: SCANNER-ANALYZERS-LANG-10-308N (Wave 2) @@ -868,9 +879,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Prereqs: SCANNER-ANALYZERS-LANG-10-303B (Wave 2) • Current: DONE — `direct_url.json` editable insights surfaced; EntryTrace usage hints mark console scripts; deterministic fixture covers editable vs wheel installs. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-306C — Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-306C — Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. • Prereqs: SCANNER-ANALYZERS-LANG-10-306B (Wave 2) - • Current: TODO + • Current: DONE — Hash fallback wired through shared helpers; fixtures ensure deterministic output. - **Sprint 12** · Runtime Guardrails - Team: Zastava Observer Guild - Path: `src/StellaOps.Zastava.Observer/TASKS.md` @@ -931,9 +942,9 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Prereqs: SCANNER-ANALYZERS-LANG-10-305C (Wave 3) • Current: DONE 2025-10-22 - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-307G — Wire shared helpers (license mapping, usage flags) and ensure concurrency-safe buffer reuse. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-307G — Wire shared helpers (license mapping, usage flags) and ensure concurrency-safe buffer reuse. • Prereqs: SCANNER-ANALYZERS-LANG-10-304C (Wave 3) - • Current: TODO + • Current: DONE — Shared helpers integrated; concurrency tests verify buffer reuse. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` 1. [TODO] SCANNER-ANALYZERS-LANG-10-307P — Shared helper integration (license metadata, quiet provenance, component merging). • Prereqs: SCANNER-ANALYZERS-LANG-10-303C (Wave 3) @@ -1003,13 +1014,13 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Prereqs: SCANNER-ANALYZERS-LANG-10-307D (Wave 4) • Current: DONE — fixtures + benchmarks merged 2025-10-23 - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-308G — Determinism fixtures + benchmark harness (Vs competitor). + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-308G — Determinism fixtures + benchmark harness (Vs competitor). • Prereqs: SCANNER-ANALYZERS-LANG-10-307G (Wave 4) - • Current: TODO + • Current: DONE — Fixtures and benchmark harness merged; perf delta captured vs competitor. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-308P — Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. + 1. [DONE 2025-10-23] SCANNER-ANALYZERS-LANG-10-308P — Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. • Prereqs: SCANNER-ANALYZERS-LANG-10-307P (Wave 4) - • Current: TODO + • Current: DONE — Fixtures `simple-venv`, `pip-cache`, `layered-editable` + hash throughput benchmarks merged 2025-10-23. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` 1. [TODO] SCANNER-ANALYZERS-LANG-10-308R — Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. • Prereqs: SCANNER-ANALYZERS-LANG-10-307R (Wave 4) @@ -1041,13 +1052,13 @@ Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster • Prereqs: SCANNER-ANALYZERS-LANG-10-308D (Wave 5) • Current: DONE — manifest + Offline Kit docs updated 2025-10-23 - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-309G — Package plug-in manifest + Offline Kit notes; ensure Worker DI registration. + 1. [DONE 2025-10-22] SCANNER-ANALYZERS-LANG-10-309G — Package plug-in manifest + Offline Kit notes; ensure Worker DI registration. • Prereqs: SCANNER-ANALYZERS-LANG-10-308G (Wave 5) - • Current: TODO + • Current: DONE — Manifest copied, Worker DI registration verified, Offline Kit docs updated. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` - 1. [TODO] SCANNER-ANALYZERS-LANG-10-309P — Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. + 1. [DONE 2025-10-23] SCANNER-ANALYZERS-LANG-10-309P — Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. • Prereqs: SCANNER-ANALYZERS-LANG-10-308P (Wave 5) - • Current: TODO + • Current: DONE — Manifest copied, Worker integration verified, Offline Kit docs updated with Python plug-in guidance. - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` 1. [TODO] SCANNER-ANALYZERS-LANG-10-309R — Package plug-in manifest + Offline Kit documentation; ensure Worker integration. • Prereqs: SCANNER-ANALYZERS-LANG-10-308R (Wave 5) diff --git a/SPRINTS.md b/SPRINTS.md index 72cd2b28..853dd81b 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -2,34 +2,10 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | | --- | --- | --- | --- | --- | --- | --- | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement layer cache store keyed by layer digest with metadata retention per architecture §3.3. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-102 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-103 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-104 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-201 | Alpine/apk analyzer emitting deterministic components with provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-202 | Debian/dpkg analyzer mapping packages to purl identity with evidence. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-203 | RPM analyzer capturing EVR, file listings, provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-204 | Shared OS evidence helpers for package identity + provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-205 | Vendor metadata enrichment (source packages, license, CVE hints). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-206 | Determinism harness + fixtures for OS analyzers. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-207 | Package OS analyzers as restart-time plug-ins (manifest + host registration). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301 | Java analyzer emitting `pkg:maven` with provenance. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | POSIX shell AST parser with deterministic output. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Command resolution across layered rootfs with evidence attribution. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Interpreter tracing for shell wrappers to Python/Node/Java launchers. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-404 | Python entry analyzer (venv shebang, module invocation, usage flag). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-405 | Node/Java launcher analyzer capturing script/jar targets. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-406 | Explainability + diagnostics for unresolved constructs with metrics. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-407 | Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-501 | Build component differ tracking add/remove/version changes with deterministic ordering. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-502 | Attribute diffs to introducing/removing layers including provenance evidence. | -| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-503 | Produce JSON diff output for inventory vs usage views aligned with API contract. | -| Sprint 10 | Samples | samples/TASKS.md | TODO | Samples Guild, Scanner Team | SAMPLES-10-001 | Sample images with SBOM/BOM-Index sidecars. | -| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-PERF-10-001 | Perf smoke job ensuring <5 s SBOM compose. | -| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DOING (2025-10-19) | Authority Core & Security Guild | AUTH-MTLS-11-002 | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-API-11-201 | `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-VERIFY-11-202 | `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-OBS-11-203 | Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. | +| Sprint 11 | Storage Platform Hardening | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-23) | Scanner Storage Guild | SCANNER-STORAGE-11-401 | Migrate scanner object storage integration from MinIO to RustFS with data migration plan. | | Sprint 11 | UI Integration | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ATTEST-11-005 | Attestation visibility (Rekor id, status) on Scan Detail. | | Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-201 | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | | Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-202 | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | @@ -57,6 +33,7 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 13 | Platform Reliability | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-NUGET-13-002 | Ensure all solutions/projects prioritize `local-nuget` before public feeds and add restore-order validation. | | Sprint 13 | Platform Reliability | ops/devops/TASKS.md | TODO | DevOps Guild, Platform Leads | DEVOPS-NUGET-13-003 | Upgrade `Microsoft.*` dependencies pinned to 8.* to their latest .NET 10 (or 9.x) releases and refresh guidance. | | Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | +| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild, Scanner Guild | DEVOPS-REL-14-004 | Extend release/offline smoke jobs to cover Python analyzer plug-ins (warm/cold, determinism, signing). | | Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | | Sprint 14 | Release & Offline Ops | ops/deployment/TASKS.md | TODO | Deployment Guild | DEVOPS-OPS-14-003 | Deployment/update/rollback automation and channel management documentation. | | Sprint 14 | Release & Offline Ops | ops/licensing/TASKS.md | TODO | Licensing Guild | DEVOPS-LIC-14-004 | Registry token service tied to Authority, plan gating, revocation handling, monitoring. | @@ -112,4 +89,5 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | | Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | | Sprint 18 | Launch Readiness | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-LAUNCH-18-001 | Production launch cutover rehearsal and runbook publication (blocked on implementation sign-off and environment setup). | -| Sprint 18 | Launch Readiness | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild, UX Specialist | DEVOPS-OFFLINE-18-003 | Capture Angular workspace npm cache + Chromium bundle for Offline Kit distribution and document refresh cadence. | \ No newline at end of file +| Sprint 18 | Launch Readiness | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild, UX Specialist | DEVOPS-OFFLINE-18-003 | Capture Angular workspace npm cache + Chromium bundle for Offline Kit distribution and document refresh cadence. | +| Sprint 18 | Launch Readiness | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild, Scanner Guild | DEVOPS-OFFLINE-18-005 | Rebuild Offline Kit with Python analyzer artefacts and refreshed manifest/signature pair. | diff --git a/SPRINTS_PRIOR_20251021.md b/SPRINTS_PRIOR_20251021.md index dc0a79ce..78e57d02 100644 --- a/SPRINTS_PRIOR_20251021.md +++ b/SPRINTS_PRIOR_20251021.md @@ -49,9 +49,37 @@ This file describe implementation of Stella Ops (docs/README.md). Implementation | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-605 | Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-606 | Usage view bit flags integrated with EntryTrace. | | Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-607 | Embed scoring inputs, confidence band, and quiet provenance in CycloneDX/DSSE artifacts. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement layer cache store keyed by layer digest with metadata retention per architecture §3.3. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-102 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-103 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-104 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-201 | Alpine/apk analyzer emitting deterministic components with provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-202 | Debian/dpkg analyzer mapping packages to purl identity with evidence. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-203 | RPM analyzer capturing EVR, file listings, provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-204 | Shared OS evidence helpers for package identity + provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-205 | Vendor metadata enrichment (source packages, license, CVE hints). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-206 | Determinism harness + fixtures for OS analyzers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-207 | Package OS analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | DONE (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301 | Java analyzer emitting `pkg:maven` with provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | POSIX shell AST parser with deterministic output. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Command resolution across layered rootfs with evidence attribution. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Interpreter tracing for shell wrappers to Python/Node/Java launchers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-404 | Python entry analyzer (venv shebang, module invocation, usage flag). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-405 | Node/Java launcher analyzer capturing script/jar targets. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-406 | Explainability + diagnostics for unresolved constructs with metrics. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-407 | Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | DONE (2025-10-19) | Diff Guild | SCANNER-DIFF-10-501 | Build component differ tracking add/remove/version changes with deterministic ordering. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | DONE (2025-10-19) | Diff Guild | SCANNER-DIFF-10-502 | Attribute diffs to introducing/removing layers including provenance evidence. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | DONE (2025-10-19) | Diff Guild | SCANNER-DIFF-10-503 | Produce JSON diff output for inventory vs usage views aligned with API contract. | +| Sprint 10 | Samples | samples/TASKS.md | DONE (2025-10-20) | Samples Guild, Scanner Team | SAMPLES-10-001 | Sample images with SBOM/BOM-Index sidecars. | +| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | DONE (2025-10-22) | DevOps Guild | DEVOPS-PERF-10-001 | Perf smoke job ensuring <5 s SBOM compose. | +| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | DONE (2025-10-23) | DevOps Guild | DEVOPS-PERF-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | +| Sprint 10 | Policy Samples | samples/TASKS.md | DONE (2025-10-23) | Samples Guild, Policy Guild | SAMPLES-13-004 | Add policy preview/report fixtures showing confidence bands and unknown-age tags. | +| Sprint 10 | Policy Samples | src/StellaOps.Web/TASKS.md | DONE (2025-10-23) | UI Guild | WEB-POLICY-FIXTURES-10-001 | Wire policy preview/report doc fixtures into UI harness (test utility or Storybook substitute) with type bindings and validation guard so UI stays aligned with documented payloads. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | DONE (2025-10-21) | Signer Guild | SIGNER-API-11-101 | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | DONE (2025-10-21) | Signer Guild | SIGNER-REF-11-102 | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | | Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | DONE (2025-10-21) | Signer Guild | SIGNER-QUOTA-11-103 | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DONE (2025-10-23) | Authority Core & Security Guild | AUTH-MTLS-11-002 | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | | Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-20) | Scanner WebService Guild | SCANNER-RUNTIME-12-301 | `/runtime/events` ingestion endpoint with validation, batching, storage hooks. | | Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | DONE (2025-10-21) | DevEx/CLI | CLI-OFFLINE-13-006 | Implement offline kit pull/import/status commands with integrity checks. | | Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | DONE (2025-10-22) | DevEx/CLI | CLI-PLUGIN-13-007 | Package non-core CLI verbs as restart-time plug-ins (manifest + loader tests). | diff --git a/bench/Scanner.Analyzers/README.md b/bench/Scanner.Analyzers/README.md index b1fa030f..2fbd6640 100644 --- a/bench/Scanner.Analyzers/README.md +++ b/bench/Scanner.Analyzers/README.md @@ -19,7 +19,10 @@ dotnet run \ --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj \ -- \ --repo-root . \ - --out bench/Scanner.Analyzers/baseline.csv + --out bench/Scanner.Analyzers/baseline.csv \ + --json out/bench/scanner-analyzers/latest.json \ + --prom out/bench/scanner-analyzers/latest.prom \ + --commit "$(git rev-parse HEAD)" ``` The harness prints a table to stdout and writes the CSV (if `--out` is specified) with the following headers: @@ -28,7 +31,16 @@ The harness prints a table to stdout and writes the CSV (if `--out` is specified scenario,iterations,sample_count,mean_ms,p95_ms,max_ms ``` -Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms (or per-scenario overrides in `config.json`), aligned with the SBOM compose objective. +Additional outputs: +- `--json` emits a deterministic report consumable by Grafana/automation (schema `1.0`, see `docs/12_PERFORMANCE_WORKBOOK.md`). +- `--prom` exports Prometheus-compatible gauges (`scanner_analyzer_bench_*`), which CI uploads for dashboards and alerts. + +Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms (or per-scenario overrides in `config.json`), aligned with the SBOM compose objective. Provide `--baseline path/to/baseline.csv` (defaults to the repo baseline) to compare against historical numbers—regressions ≥ 20 % on the `max_ms` metric or breaches of the configured threshold will fail the run. + +Metadata options: +- `--captured-at 2025-10-23T12:00:00Z` to inject a deterministic timestamp (otherwise `UtcNow`). +- `--commit` and `--environment` annotate the JSON report for dashboards. +- `--regression-limit 1.15` adjusts the ratio guard (default 1.20 ⇒ +20 %). ## Adding scenarios 1. Drop the fixture tree under `samples//...`. @@ -38,5 +50,5 @@ Use `--iterations` to override the default (5 passes per scenario) and `--thresh - `root` – path to the directory that will be scanned. - For analyzer-backed scenarios, set `analyzers` to the list of language analyzer ids (for example, `["node"]`). - For temporary metadata walks (used until the analyzer ships), provide `parser` (`node` or `python`) and the `matcher` glob describing files to parse. -3. Re-run the harness (`dotnet run … --out baseline.csv`). +3. Re-run the harness (`dotnet run … --out baseline.csv --json out/.../new.json --prom out/.../new.prom`). 4. Commit both the fixture and updated baseline. diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BaselineLoaderTests.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BaselineLoaderTests.cs new file mode 100644 index 00000000..b479eb9a --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BaselineLoaderTests.cs @@ -0,0 +1,37 @@ +using System.Text; +using StellaOps.Bench.ScannerAnalyzers.Baseline; +using Xunit; + +namespace StellaOps.Bench.ScannerAnalyzers.Tests; + +public sealed class BaselineLoaderTests +{ + [Fact] + public async Task LoadAsync_ReadsCsvIntoDictionary() + { + var csv = """ + scenario,iterations,sample_count,mean_ms,p95_ms,max_ms + node_monorepo_walk,5,4,9.4303,36.1354,45.0012 + python_site_packages_walk,5,10,12.1000,18.2000,26.3000 + """; + + var path = await WriteTempFileAsync(csv); + + var result = await BaselineLoader.LoadAsync(path, CancellationToken.None); + + Assert.Equal(2, result.Count); + var entry = Assert.Contains("node_monorepo_walk", result); + Assert.Equal(5, entry.Iterations); + Assert.Equal(4, entry.SampleCount); + Assert.Equal(9.4303, entry.MeanMs, 4); + Assert.Equal(36.1354, entry.P95Ms, 4); + Assert.Equal(45.0012, entry.MaxMs, 4); + } + + private static async Task WriteTempFileAsync(string content) + { + var path = Path.Combine(Path.GetTempPath(), $"baseline-{Guid.NewGuid():N}.csv"); + await File.WriteAllTextAsync(path, content, Encoding.UTF8); + return path; + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkJsonWriterTests.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkJsonWriterTests.cs new file mode 100644 index 00000000..6fd4c713 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkJsonWriterTests.cs @@ -0,0 +1,41 @@ +using System.Text.Json; +using StellaOps.Bench.ScannerAnalyzers; +using StellaOps.Bench.ScannerAnalyzers.Baseline; +using StellaOps.Bench.ScannerAnalyzers.Reporting; +using Xunit; + +namespace StellaOps.Bench.ScannerAnalyzers.Tests; + +public sealed class BenchmarkJsonWriterTests +{ + [Fact] + public async Task WriteAsync_EmitsMetadataAndScenarioDetails() + { + var metadata = new BenchmarkJsonMetadata("1.0", DateTimeOffset.Parse("2025-10-23T12:00:00Z"), "abc123", "ci"); + var result = new ScenarioResult( + "scenario", + "Scenario", + SampleCount: 5, + MeanMs: 10, + P95Ms: 12, + MaxMs: 20, + Iterations: 5, + ThresholdMs: 5000); + var baseline = new BaselineEntry("scenario", 5, 5, 9, 11, 10); + var report = new BenchmarkScenarioReport(result, baseline, 1.2); + + var path = Path.Combine(Path.GetTempPath(), $"bench-{Guid.NewGuid():N}.json"); + await BenchmarkJsonWriter.WriteAsync(path, metadata, new[] { report }, CancellationToken.None); + + using var document = JsonDocument.Parse(await File.ReadAllTextAsync(path)); + var root = document.RootElement; + + Assert.Equal("1.0", root.GetProperty("schemaVersion").GetString()); + Assert.Equal("abc123", root.GetProperty("commit").GetString()); + var scenario = root.GetProperty("scenarios")[0]; + Assert.Equal("scenario", scenario.GetProperty("id").GetString()); + Assert.Equal(20, scenario.GetProperty("maxMs").GetDouble()); + Assert.Equal(10, scenario.GetProperty("baseline").GetProperty("maxMs").GetDouble()); + Assert.True(scenario.GetProperty("regression").GetProperty("breached").GetBoolean()); + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkScenarioReportTests.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkScenarioReportTests.cs new file mode 100644 index 00000000..e0da853a --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkScenarioReportTests.cs @@ -0,0 +1,58 @@ +using StellaOps.Bench.ScannerAnalyzers; +using StellaOps.Bench.ScannerAnalyzers.Baseline; +using StellaOps.Bench.ScannerAnalyzers.Reporting; +using Xunit; + +namespace StellaOps.Bench.ScannerAnalyzers.Tests; + +public sealed class BenchmarkScenarioReportTests +{ + [Fact] + public void RegressionRatio_ComputedWhenBaselinePresent() + { + var result = new ScenarioResult( + "scenario", + "Scenario", + SampleCount: 5, + MeanMs: 10, + P95Ms: 12, + MaxMs: 20, + Iterations: 5, + ThresholdMs: 5000); + + var baseline = new BaselineEntry( + "scenario", + Iterations: 5, + SampleCount: 5, + MeanMs: 8, + P95Ms: 11, + MaxMs: 15); + + var report = new BenchmarkScenarioReport(result, baseline, regressionLimit: 1.2); + + Assert.True(report.MaxRegressionRatio.HasValue); + Assert.Equal(20d / 15d, report.MaxRegressionRatio.Value, 6); + Assert.True(report.RegressionBreached); + Assert.Contains("+33.3%", report.BuildRegressionFailureMessage()); + } + + [Fact] + public void RegressionRatio_NullWhenBaselineMissing() + { + var result = new ScenarioResult( + "scenario", + "Scenario", + SampleCount: 5, + MeanMs: 10, + P95Ms: 12, + MaxMs: 20, + Iterations: 5, + ThresholdMs: 5000); + + var report = new BenchmarkScenarioReport(result, baseline: null, regressionLimit: 1.2); + + Assert.Null(report.MaxRegressionRatio); + Assert.False(report.RegressionBreached); + Assert.Null(report.BuildRegressionFailureMessage()); + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/PrometheusWriterTests.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/PrometheusWriterTests.cs new file mode 100644 index 00000000..0e1dfd64 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/PrometheusWriterTests.cs @@ -0,0 +1,32 @@ +using StellaOps.Bench.ScannerAnalyzers; +using StellaOps.Bench.ScannerAnalyzers.Baseline; +using StellaOps.Bench.ScannerAnalyzers.Reporting; +using Xunit; + +namespace StellaOps.Bench.ScannerAnalyzers.Tests; + +public sealed class PrometheusWriterTests +{ + [Fact] + public void Write_EmitsMetricsForScenario() + { + var result = new ScenarioResult( + "scenario_a", + "Scenario A", + SampleCount: 5, + MeanMs: 10, + P95Ms: 12, + MaxMs: 20, + Iterations: 5, + ThresholdMs: 5000); + var baseline = new BaselineEntry("scenario_a", 5, 5, 9, 11, 18); + var report = new BenchmarkScenarioReport(result, baseline, 1.2); + + var path = Path.Combine(Path.GetTempPath(), $"metrics-{Guid.NewGuid():N}.prom"); + PrometheusWriter.Write(path, new[] { report }); + + var contents = File.ReadAllText(path); + Assert.Contains("scanner_analyzer_bench_max_ms{scenario=\"scenario_a\"} 20", contents); + Assert.Contains("scanner_analyzer_bench_regression_ratio{scenario=\"scenario_a\"}", contents); + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/StellaOps.Bench.ScannerAnalyzers.Tests.csproj b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/StellaOps.Bench.ScannerAnalyzers.Tests.csproj new file mode 100644 index 00000000..e2ccb99b --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/StellaOps.Bench.ScannerAnalyzers.Tests.csproj @@ -0,0 +1,26 @@ + + + net10.0 + enable + enable + preview + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Baseline/BaselineEntry.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Baseline/BaselineEntry.cs new file mode 100644 index 00000000..37e69949 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Baseline/BaselineEntry.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Bench.ScannerAnalyzers.Baseline; + +internal sealed record BaselineEntry( + string ScenarioId, + int Iterations, + int SampleCount, + double MeanMs, + double P95Ms, + double MaxMs); diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Baseline/BaselineLoader.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Baseline/BaselineLoader.cs new file mode 100644 index 00000000..db39b57e --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Baseline/BaselineLoader.cs @@ -0,0 +1,88 @@ +using System.Globalization; + +namespace StellaOps.Bench.ScannerAnalyzers.Baseline; + +internal static class BaselineLoader +{ + public static async Task> LoadAsync(string path, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("Baseline path must be provided.", nameof(path)); + } + + var resolved = Path.GetFullPath(path); + if (!File.Exists(resolved)) + { + throw new FileNotFoundException($"Baseline file not found at {resolved}", resolved); + } + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await using var stream = new FileStream(resolved, FileMode.Open, FileAccess.Read, FileShare.Read); + using var reader = new StreamReader(stream); + string? line; + var isFirst = true; + + while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) + { + cancellationToken.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + if (isFirst) + { + isFirst = false; + if (line.StartsWith("scenario,", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + } + + var entry = ParseLine(line); + result[entry.ScenarioId] = entry; + } + + return result; + } + + private static BaselineEntry ParseLine(string line) + { + var parts = line.Split(',', StringSplitOptions.TrimEntries); + if (parts.Length < 6) + { + throw new InvalidDataException($"Baseline CSV row malformed: '{line}'"); + } + + var scenarioId = parts[0]; + var iterations = ParseInt(parts[1], nameof(BaselineEntry.Iterations)); + var sampleCount = ParseInt(parts[2], nameof(BaselineEntry.SampleCount)); + var meanMs = ParseDouble(parts[3], nameof(BaselineEntry.MeanMs)); + var p95Ms = ParseDouble(parts[4], nameof(BaselineEntry.P95Ms)); + var maxMs = ParseDouble(parts[5], nameof(BaselineEntry.MaxMs)); + + return new BaselineEntry(scenarioId, iterations, sampleCount, meanMs, p95Ms, maxMs); + } + + private static int ParseInt(string value, string field) + { + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + throw new InvalidDataException($"Failed to parse integer {field} from '{value}'."); + } + + return parsed; + } + + private static double ParseDouble(string value, string field) + { + if (!double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) + { + throw new InvalidDataException($"Failed to parse double {field} from '{value}'."); + } + + return parsed; + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs index 353c99f2..0100de0b 100644 --- a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs @@ -1,4 +1,6 @@ using System.Globalization; +using StellaOps.Bench.ScannerAnalyzers.Baseline; +using StellaOps.Bench.ScannerAnalyzers.Reporting; using StellaOps.Bench.ScannerAnalyzers.Scenarios; namespace StellaOps.Bench.ScannerAnalyzers; @@ -15,8 +17,13 @@ internal static class Program var iterations = options.Iterations ?? config.Iterations ?? 5; var thresholdMs = options.ThresholdMs ?? config.ThresholdMs ?? 5000; var repoRoot = ResolveRepoRoot(options.RepoRoot, options.ConfigPath); + var regressionLimit = options.RegressionLimit ?? 1.2d; + var capturedAt = (options.CapturedAtUtc ?? DateTimeOffset.UtcNow).ToUniversalTime(); + + var baseline = await LoadBaselineDictionaryAsync(options.BaselinePath, CancellationToken.None).ConfigureAwait(false); var results = new List(); + var reports = new List(); var failures = new List(); foreach (var scenario in config.Scenarios) @@ -28,26 +35,54 @@ internal static class Program var stats = ScenarioStatistics.FromDurations(execution.Durations); var scenarioThreshold = scenario.ThresholdMs ?? thresholdMs; - results.Add(new ScenarioResult( + var result = new ScenarioResult( scenario.Id!, scenario.Label ?? scenario.Id!, execution.SampleCount, stats.MeanMs, stats.P95Ms, stats.MaxMs, - iterations)); + iterations, + scenarioThreshold); + + results.Add(result); if (stats.MaxMs > scenarioThreshold) { failures.Add($"{scenario.Id} exceeded threshold: {stats.MaxMs:F2} ms > {scenarioThreshold:F2} ms"); } + + baseline.TryGetValue(result.Id, out var baselineEntry); + var report = new BenchmarkScenarioReport(result, baselineEntry, regressionLimit); + if (report.BuildRegressionFailureMessage() is { } regressionFailure) + { + failures.Add(regressionFailure); + } + + reports.Add(report); } TablePrinter.Print(results); - if (!string.IsNullOrWhiteSpace(options.OutPath)) + if (!string.IsNullOrWhiteSpace(options.CsvOutPath)) { - CsvWriter.Write(options.OutPath!, results); + CsvWriter.Write(options.CsvOutPath!, results); + } + + if (!string.IsNullOrWhiteSpace(options.JsonOutPath)) + { + var metadata = new BenchmarkJsonMetadata( + "1.0", + capturedAt, + options.Commit, + options.Environment); + + await BenchmarkJsonWriter.WriteAsync(options.JsonOutPath!, metadata, reports, CancellationToken.None).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(options.PrometheusOutPath)) + { + PrometheusWriter.Write(options.PrometheusOutPath!, reports); } if (failures.Count > 0) @@ -71,6 +106,22 @@ internal static class Program } } + private static async Task> LoadBaselineDictionaryAsync(string? baselinePath, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(baselinePath)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var resolved = Path.GetFullPath(baselinePath); + if (!File.Exists(resolved)) + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + return await BaselineLoader.LoadAsync(resolved, cancellationToken).ConfigureAwait(false); + } + private static string ResolveRepoRoot(string? overridePath, string configPath) { if (!string.IsNullOrWhiteSpace(overridePath)) @@ -108,15 +159,34 @@ internal static class Program return combined; } - private sealed record ProgramOptions(string ConfigPath, int? Iterations, double? ThresholdMs, string? OutPath, string? RepoRoot) + private sealed record ProgramOptions( + string ConfigPath, + int? Iterations, + double? ThresholdMs, + string? CsvOutPath, + string? JsonOutPath, + string? PrometheusOutPath, + string? RepoRoot, + string? BaselinePath, + DateTimeOffset? CapturedAtUtc, + string? Commit, + string? Environment, + double? RegressionLimit) { public static ProgramOptions Parse(string[] args) { var configPath = DefaultConfigPath(); + var baselinePath = DefaultBaselinePath(); int? iterations = null; double? thresholdMs = null; - string? outPath = null; + string? csvOut = null; + string? jsonOut = null; + string? promOut = null; string? repoRoot = null; + DateTimeOffset? capturedAt = null; + string? commit = null; + string? environment = null; + double? regressionLimit = null; for (var index = 0; index < args.Length; index++) { @@ -136,20 +206,50 @@ internal static class Program thresholdMs = double.Parse(args[++index], CultureInfo.InvariantCulture); break; case "--out": + case "--csv": EnsureNext(args, index); - outPath = args[++index]; + csvOut = args[++index]; + break; + case "--json": + EnsureNext(args, index); + jsonOut = args[++index]; + break; + case "--prom": + case "--prometheus": + EnsureNext(args, index); + promOut = args[++index]; + break; + case "--baseline": + EnsureNext(args, index); + baselinePath = args[++index]; break; case "--repo-root": case "--samples": EnsureNext(args, index); repoRoot = args[++index]; break; + case "--captured-at": + EnsureNext(args, index); + capturedAt = DateTimeOffset.Parse(args[++index], CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + break; + case "--commit": + EnsureNext(args, index); + commit = args[++index]; + break; + case "--environment": + EnsureNext(args, index); + environment = args[++index]; + break; + case "--regression-limit": + EnsureNext(args, index); + regressionLimit = double.Parse(args[++index], CultureInfo.InvariantCulture); + break; default: throw new ArgumentException($"Unknown argument: {current}", nameof(args)); } } - return new ProgramOptions(configPath, iterations, thresholdMs, outPath, repoRoot); + return new ProgramOptions(configPath, iterations, thresholdMs, csvOut, jsonOut, promOut, repoRoot, baselinePath, capturedAt, commit, environment, regressionLimit); } private static string DefaultConfigPath() @@ -160,6 +260,15 @@ internal static class Program return Path.Combine(configDirectory, "config.json"); } + private static string? DefaultBaselinePath() + { + var binaryDir = AppContext.BaseDirectory; + var projectRoot = Path.GetFullPath(Path.Combine(binaryDir, "..", "..", "..")); + var benchRoot = Path.GetFullPath(Path.Combine(projectRoot, "..")); + var baselinePath = Path.Combine(benchRoot, "baseline.csv"); + return File.Exists(baselinePath) ? baselinePath : baselinePath; + } + private static void EnsureNext(string[] args, int index) { if (index + 1 >= args.Length) @@ -169,15 +278,6 @@ internal static class Program } } - private sealed record ScenarioResult( - string Id, - string Label, - int SampleCount, - double MeanMs, - double P95Ms, - double MaxMs, - int Iterations); - private sealed record ScenarioStatistics(double MeanMs, double P95Ms, double MaxMs) { public static ScenarioStatistics FromDurations(IReadOnlyList durations) @@ -232,25 +332,16 @@ internal static class Program Console.WriteLine("---------------------------- | ----- | --------- | --------- | ----------"); foreach (var row in results) { - Console.WriteLine(FormatRow(row)); + Console.WriteLine(string.Join(" | ", new[] + { + row.IdColumn, + row.SampleCountColumn, + row.MeanColumn, + row.P95Column, + row.MaxColumn + })); } } - - private static string FormatRow(ScenarioResult row) - { - var idColumn = row.Id.Length <= 28 - ? row.Id.PadRight(28) - : row.Id[..28]; - - return string.Join(" | ", new[] - { - idColumn, - row.SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5), - row.MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9), - row.P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9), - row.MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10), - }); - } } private static class CsvWriter diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs new file mode 100644 index 00000000..183415b6 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs @@ -0,0 +1,108 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Bench.ScannerAnalyzers.Baseline; + +namespace StellaOps.Bench.ScannerAnalyzers.Reporting; + +internal static class BenchmarkJsonWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + public static async Task WriteAsync( + string path, + BenchmarkJsonMetadata metadata, + IReadOnlyList reports, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(metadata); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var document = new BenchmarkJsonDocument( + metadata.SchemaVersion, + metadata.CapturedAtUtc, + metadata.Commit, + metadata.Environment, + reports.Select(CreateScenario).ToArray()); + + await using var stream = new FileStream(resolved, FileMode.Create, FileAccess.Write, FileShare.None); + await JsonSerializer.SerializeAsync(stream, document, SerializerOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static BenchmarkJsonScenario CreateScenario(BenchmarkScenarioReport report) + { + var baseline = report.Baseline; + return new BenchmarkJsonScenario( + report.Result.Id, + report.Result.Label, + report.Result.Iterations, + report.Result.SampleCount, + report.Result.MeanMs, + report.Result.P95Ms, + report.Result.MaxMs, + report.Result.ThresholdMs, + baseline is null + ? null + : new BenchmarkJsonScenarioBaseline( + baseline.Iterations, + baseline.SampleCount, + baseline.MeanMs, + baseline.P95Ms, + baseline.MaxMs), + new BenchmarkJsonScenarioRegression( + report.MaxRegressionRatio, + report.MeanRegressionRatio, + report.RegressionLimit, + report.RegressionBreached)); + } + + private sealed record BenchmarkJsonDocument( + string SchemaVersion, + DateTimeOffset CapturedAt, + string? Commit, + string? Environment, + IReadOnlyList Scenarios); + + private sealed record BenchmarkJsonScenario( + string Id, + string Label, + int Iterations, + int SampleCount, + double MeanMs, + double P95Ms, + double MaxMs, + double ThresholdMs, + BenchmarkJsonScenarioBaseline? Baseline, + BenchmarkJsonScenarioRegression Regression); + + private sealed record BenchmarkJsonScenarioBaseline( + int Iterations, + int SampleCount, + double MeanMs, + double P95Ms, + double MaxMs); + + private sealed record BenchmarkJsonScenarioRegression( + double? MaxRatio, + double? MeanRatio, + double Limit, + bool Breached); +} + +internal sealed record BenchmarkJsonMetadata( + string SchemaVersion, + DateTimeOffset CapturedAtUtc, + string? Commit, + string? Environment); diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs new file mode 100644 index 00000000..55ab4ba4 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs @@ -0,0 +1,55 @@ +using StellaOps.Bench.ScannerAnalyzers.Baseline; + +namespace StellaOps.Bench.ScannerAnalyzers.Reporting; + +internal sealed class BenchmarkScenarioReport +{ + private const double RegressionLimitDefault = 1.2d; + + public BenchmarkScenarioReport(ScenarioResult result, BaselineEntry? baseline, double? regressionLimit = null) + { + Result = result ?? throw new ArgumentNullException(nameof(result)); + Baseline = baseline; + RegressionLimit = regressionLimit is { } limit && limit > 0 ? limit : RegressionLimitDefault; + MaxRegressionRatio = CalculateRatio(result.MaxMs, baseline?.MaxMs); + MeanRegressionRatio = CalculateRatio(result.MeanMs, baseline?.MeanMs); + } + + public ScenarioResult Result { get; } + + public BaselineEntry? Baseline { get; } + + public double RegressionLimit { get; } + + public double? MaxRegressionRatio { get; } + + public double? MeanRegressionRatio { get; } + + public bool RegressionBreached => MaxRegressionRatio.HasValue && MaxRegressionRatio.Value >= RegressionLimit; + + public string? BuildRegressionFailureMessage() + { + if (!RegressionBreached || MaxRegressionRatio is null) + { + return null; + } + + var percentage = (MaxRegressionRatio.Value - 1d) * 100d; + return $"{Result.Id} exceeded regression budget: max {Result.MaxMs:F2} ms vs baseline {Baseline!.MaxMs:F2} ms (+{percentage:F1}%)"; + } + + private static double? CalculateRatio(double current, double? baseline) + { + if (!baseline.HasValue) + { + return null; + } + + if (baseline.Value <= 0d) + { + return null; + } + + return current / baseline.Value; + } +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs new file mode 100644 index 00000000..03697ff5 --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs @@ -0,0 +1,59 @@ +using System.Globalization; +using System.Text; + +namespace StellaOps.Bench.ScannerAnalyzers.Reporting; + +internal static class PrometheusWriter +{ + public static void Write(string path, IReadOnlyList reports) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ArgumentNullException.ThrowIfNull(reports); + + var resolved = Path.GetFullPath(path); + var directory = Path.GetDirectoryName(resolved); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var builder = new StringBuilder(); + builder.AppendLine("# HELP scanner_analyzer_bench_duration_ms Analyzer benchmark duration metrics in milliseconds."); + builder.AppendLine("# TYPE scanner_analyzer_bench_duration_ms gauge"); + + foreach (var report in reports) + { + var scenarioLabel = Escape(report.Result.Id); + AppendMetric(builder, "scanner_analyzer_bench_mean_ms", scenarioLabel, report.Result.MeanMs); + AppendMetric(builder, "scanner_analyzer_bench_p95_ms", scenarioLabel, report.Result.P95Ms); + AppendMetric(builder, "scanner_analyzer_bench_max_ms", scenarioLabel, report.Result.MaxMs); + AppendMetric(builder, "scanner_analyzer_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs); + + if (report.Baseline is { } baseline) + { + AppendMetric(builder, "scanner_analyzer_bench_baseline_max_ms", scenarioLabel, baseline.MaxMs); + AppendMetric(builder, "scanner_analyzer_bench_baseline_mean_ms", scenarioLabel, baseline.MeanMs); + } + + if (report.MaxRegressionRatio is { } ratio) + { + AppendMetric(builder, "scanner_analyzer_bench_regression_ratio", scenarioLabel, ratio); + AppendMetric(builder, "scanner_analyzer_bench_regression_limit", scenarioLabel, report.RegressionLimit); + AppendMetric(builder, "scanner_analyzer_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0); + } + } + + File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8); + } + + private static void AppendMetric(StringBuilder builder, string metric, string scenarioLabel, double value) + { + builder.Append(metric); + builder.Append("{scenario=\""); + builder.Append(scenarioLabel); + builder.Append("\"} "); + builder.AppendLine(value.ToString("G17", CultureInfo.InvariantCulture)); + } + + private static string Escape(string value) => value.Replace("\\", "\\\\", StringComparison.Ordinal).Replace("\"", "\\\"", StringComparison.Ordinal); +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs new file mode 100644 index 00000000..4632bb6c --- /dev/null +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs @@ -0,0 +1,24 @@ +using System.Globalization; + +namespace StellaOps.Bench.ScannerAnalyzers; + +internal sealed record ScenarioResult( + string Id, + string Label, + int SampleCount, + double MeanMs, + double P95Ms, + double MaxMs, + int Iterations, + double ThresholdMs) +{ + public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28]; + + public string SampleCountColumn => SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5); + + public string MeanColumn => MeanMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9); + + public string P95Column => P95Ms.ToString("F2", CultureInfo.InvariantCulture).PadLeft(9); + + public string MaxColumn => MaxMs.ToString("F2", CultureInfo.InvariantCulture).PadLeft(10); +} diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs index 2f3c72a7..bbbb4a3b 100644 --- a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs @@ -8,6 +8,7 @@ using StellaOps.Scanner.Analyzers.Lang.Go; using StellaOps.Scanner.Analyzers.Lang.Java; using StellaOps.Scanner.Analyzers.Lang.Node; using StellaOps.Scanner.Analyzers.Lang.DotNet; +using StellaOps.Scanner.Analyzers.Lang.Python; namespace StellaOps.Bench.ScannerAnalyzers.Scenarios; @@ -109,6 +110,7 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner "go" => static () => new GoLanguageAnalyzer(), "node" => static () => new NodeLanguageAnalyzer(), "dotnet" => static () => new DotNetLanguageAnalyzer(), + "python" => static () => new PythonLanguageAnalyzer(), _ => throw new InvalidOperationException($"Unsupported analyzer '{analyzerId}'."), }; } diff --git a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj index 04f0e626..0870250e 100644 --- a/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj +++ b/bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj @@ -14,5 +14,10 @@ + + + + + diff --git a/bench/Scanner.Analyzers/baseline.csv b/bench/Scanner.Analyzers/baseline.csv index fac7745d..a75ee90e 100644 --- a/bench/Scanner.Analyzers/baseline.csv +++ b/bench/Scanner.Analyzers/baseline.csv @@ -1,6 +1,7 @@ scenario,iterations,sample_count,mean_ms,p95_ms,max_ms -node_monorepo_walk,5,4,9.4303,36.1354,45.0012 -java_demo_archive,5,1,20.6964,81.5592,101.7846 -go_buildinfo_fixture,5,2,35.0345,136.5466,170.1612 -dotnet_multirid_fixture,5,2,29.1862,106.6249,132.3018 -python_site_packages_walk,5,3,12.0024,45.0165,56.0003 +node_monorepo_walk,5,4,6.0975,21.7421,26.8537 +java_demo_archive,5,1,6.2007,23.4837,29.1143 +go_buildinfo_fixture,5,2,6.1949,22.6851,27.9196 +dotnet_multirid_fixture,5,2,11.4884,37.7460,46.4850 +python_site_packages_scan,5,3,5.6420,18.2943,22.3739 +python_pip_cache_fixture,5,1,5.8598,13.2855,15.6256 diff --git a/bench/Scanner.Analyzers/config.json b/bench/Scanner.Analyzers/config.json index 45c1d3d2..7926837f 100644 --- a/bench/Scanner.Analyzers/config.json +++ b/bench/Scanner.Analyzers/config.json @@ -35,11 +35,20 @@ ] }, { - "id": "python_site_packages_walk", - "label": "Python site-packages dist-info crawl", - "root": "samples/runtime/python-venv/lib/python3.11/site-packages", - "matcher": "**/*.dist-info/METADATA", - "parser": "python" - } - ] -} + "id": "python_site_packages_scan", + "label": "Python analyzer on sample virtualenv", + "root": "samples/runtime/python-venv", + "analyzers": [ + "python" + ] + }, + { + "id": "python_pip_cache_fixture", + "label": "Python analyzer verifying RECORD hashes", + "root": "src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache", + "analyzers": [ + "python" + ] + } + ] +} diff --git a/bench/Scanner.Analyzers/lang/README.md b/bench/Scanner.Analyzers/lang/README.md index 5aeb2671..3e166861 100644 --- a/bench/Scanner.Analyzers/lang/README.md +++ b/bench/Scanner.Analyzers/lang/README.md @@ -23,3 +23,9 @@ Results should be committed as deterministic CSV/JSON outputs with accompanying - Scenario `dotnet_multirid_fixture` exercises the .NET analyzer against the multi-RID test fixture that merges two applications and four runtime identifiers. Latest baseline run (Release build, 5 iterations) records a mean duration of **29.19 ms** (p95 106.62 ms, max 132.30 ms) with a stable component count of 2. - Syft v1.29.1 scanning the same fixture (`syft scan dir:…`) averaged **1 546 ms** (p95 ≈2 100 ms, max ≈2 100 ms) while also reporting duplicate packages; raw numbers captured in `dotnet/syft-comparison-20251023.csv`. - The new scenario is declared in `bench/Scanner.Analyzers/config.json`; rerun the bench command above after rebuilding analyzers to refresh baselines and comparison data. + +## Sprint LA2 — Python Analyzer Benchmark Notes (2025-10-23) + +- Added two Python scenarios to `config.json`: the virtualenv sample (`python_site_packages_scan`) and the RECORD-heavy pip cache fixture (`python_pip_cache_fixture`). +- Baseline run (Release build, 5 iterations) records means of **5.64 ms** (p95 18.29 ms) for the virtualenv and **5.86 ms** (p95 13.29 ms) for the pip cache verifier; raw numbers stored in `python/hash-throughput-20251023.csv`. +- The pip cache fixture exercises `PythonRecordVerifier` with 12 RECORD rows (7 hashed) and mismatched layer coverage, giving a repeatable hash-validation throughput reference for regression gating. diff --git a/bench/Scanner.Analyzers/lang/python/hash-throughput-20251023.csv b/bench/Scanner.Analyzers/lang/python/hash-throughput-20251023.csv new file mode 100644 index 00000000..fd965df4 --- /dev/null +++ b/bench/Scanner.Analyzers/lang/python/hash-throughput-20251023.csv @@ -0,0 +1,3 @@ +scenario,iterations,sample_count,mean_ms,p95_ms,max_ms +python_site_packages_scan,5,3,5.6420,18.2943,22.3739 +python_pip_cache_fixture,5,1,5.8598,13.2855,15.6256 diff --git a/deploy/compose/docker-compose.airgap.yaml b/deploy/compose/docker-compose.airgap.yaml index 8930ba98..44a9df95 100644 --- a/deploy/compose/docker-compose.airgap.yaml +++ b/deploy/compose/docker-compose.airgap.yaml @@ -7,11 +7,12 @@ networks: stellaops: driver: bridge -volumes: - mongo-data: - minio-data: - concelier-jobs: - nats-data: +volumes: + mongo-data: + minio-data: + rustfs-data: + concelier-jobs: + nats-data: services: mongo: @@ -27,8 +28,8 @@ services: - stellaops labels: *release-labels - minio: - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e command: ["server", "/data", "--console-address", ":9001"] restart: unless-stopped environment: @@ -40,7 +41,22 @@ services: - "${MINIO_CONSOLE_PORT:-29001}:9001" networks: - stellaops - labels: *release-labels + labels: *release-labels + + rustfs: + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"] + restart: unless-stopped + environment: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumes: + - rustfs-data:/data + ports: + - "${RUSTFS_HTTP_PORT:-8080}:8080" + networks: + - stellaops + labels: *release-labels nats: image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e @@ -127,18 +143,19 @@ services: - stellaops labels: *release-labels - scanner-web: + scanner-web: image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 restart: unless-stopped - depends_on: - - concelier - - minio - - nats - environment: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + depends_on: + - concelier + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}" SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" @@ -152,17 +169,19 @@ services: - stellaops labels: *release-labels - scanner-worker: + scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 restart: unless-stopped - depends_on: - - scanner-web - - nats - environment: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + depends_on: + - scanner-web + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" networks: - stellaops diff --git a/deploy/compose/docker-compose.dev.yaml b/deploy/compose/docker-compose.dev.yaml index 6f582886..81a2434a 100644 --- a/deploy/compose/docker-compose.dev.yaml +++ b/deploy/compose/docker-compose.dev.yaml @@ -7,11 +7,12 @@ networks: stellaops: driver: bridge -volumes: - mongo-data: - minio-data: - concelier-jobs: - nats-data: +volumes: + mongo-data: + minio-data: + rustfs-data: + concelier-jobs: + nats-data: services: mongo: @@ -27,9 +28,9 @@ services: - stellaops labels: *release-labels - minio: - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e - command: ["server", "/data", "--console-address", ":9001"] + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data", "--console-address", ":9001"] restart: unless-stopped environment: MINIO_ROOT_USER: "${MINIO_ROOT_USER}" @@ -38,9 +39,24 @@ services: - minio-data:/data ports: - "${MINIO_CONSOLE_PORT:-9001}:9001" - networks: - - stellaops - labels: *release-labels + networks: + - stellaops + labels: *release-labels + + rustfs: + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"] + restart: unless-stopped + environment: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumes: + - rustfs-data:/data + ports: + - "${RUSTFS_HTTP_PORT:-8080}:8080" + networks: + - stellaops + labels: *release-labels nats: image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e @@ -125,18 +141,19 @@ services: - stellaops labels: *release-labels - scanner-web: + scanner-web: image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 restart: unless-stopped - depends_on: - - concelier - - minio - - nats - environment: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + depends_on: + - concelier + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}" SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" @@ -150,18 +167,20 @@ services: - stellaops labels: *release-labels - scanner-worker: + scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 restart: unless-stopped - depends_on: - - scanner-web - - nats - environment: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" - SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + depends_on: + - scanner-web + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" networks: - stellaops labels: *release-labels diff --git a/deploy/compose/docker-compose.stage.yaml b/deploy/compose/docker-compose.stage.yaml index a5f68028..259df302 100644 --- a/deploy/compose/docker-compose.stage.yaml +++ b/deploy/compose/docker-compose.stage.yaml @@ -7,11 +7,12 @@ networks: stellaops: driver: bridge -volumes: - mongo-data: - minio-data: - concelier-jobs: - nats-data: +volumes: + mongo-data: + minio-data: + rustfs-data: + concelier-jobs: + nats-data: services: mongo: @@ -27,8 +28,8 @@ services: - stellaops labels: *release-labels - minio: - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e command: ["server", "/data", "--console-address", ":9001"] restart: unless-stopped environment: @@ -40,7 +41,22 @@ services: - "${MINIO_CONSOLE_PORT:-9001}:9001" networks: - stellaops - labels: *release-labels + labels: *release-labels + + rustfs: + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"] + restart: unless-stopped + environment: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumes: + - rustfs-data:/data + ports: + - "${RUSTFS_HTTP_PORT:-8080}:8080" + networks: + - stellaops + labels: *release-labels nats: image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e @@ -125,18 +141,19 @@ services: - stellaops labels: *release-labels - scanner-web: + scanner-web: image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 restart: unless-stopped - depends_on: - - concelier - - minio - - nats - environment: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + depends_on: + - concelier + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}" SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}" @@ -150,17 +167,19 @@ services: - stellaops labels: *release-labels - scanner-worker: + scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab restart: unless-stopped - depends_on: - - scanner-web - - nats - environment: - SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + depends_on: + - scanner-web + - rustfs + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" networks: - stellaops diff --git a/deploy/compose/env/airgap.env.example b/deploy/compose/env/airgap.env.example index 11190d3f..e0d2662a 100644 --- a/deploy/compose/env/airgap.env.example +++ b/deploy/compose/env/airgap.env.example @@ -2,9 +2,10 @@ MONGO_INITDB_ROOT_USERNAME=stellaops MONGO_INITDB_ROOT_PASSWORD=airgap-password MINIO_ROOT_USER=stellaops-offline -MINIO_ROOT_PASSWORD=airgap-minio-secret -MINIO_CONSOLE_PORT=29001 -AUTHORITY_ISSUER=https://authority.airgap.local +MINIO_ROOT_PASSWORD=airgap-minio-secret +MINIO_CONSOLE_PORT=29001 +RUSTFS_HTTP_PORT=8080 +AUTHORITY_ISSUER=https://authority.airgap.local AUTHORITY_PORT=8440 SIGNER_POE_INTROSPECT_URL=file:///offline/poe/introspect.json SIGNER_PORT=8441 diff --git a/deploy/compose/env/dev.env.example b/deploy/compose/env/dev.env.example index af218de8..9c955751 100644 --- a/deploy/compose/env/dev.env.example +++ b/deploy/compose/env/dev.env.example @@ -3,8 +3,9 @@ MONGO_INITDB_ROOT_USERNAME=stellaops MONGO_INITDB_ROOT_PASSWORD=dev-password MINIO_ROOT_USER=stellaops MINIO_ROOT_PASSWORD=dev-minio-secret -MINIO_CONSOLE_PORT=9001 -AUTHORITY_ISSUER=https://authority.localtest.me +MINIO_CONSOLE_PORT=9001 +RUSTFS_HTTP_PORT=8080 +AUTHORITY_ISSUER=https://authority.localtest.me AUTHORITY_PORT=8440 SIGNER_POE_INTROSPECT_URL=https://licensing.svc.local/introspect SIGNER_PORT=8441 diff --git a/deploy/compose/env/mirror.env.example b/deploy/compose/env/mirror.env.example index d848cbd6..13cea5af 100644 --- a/deploy/compose/env/mirror.env.example +++ b/deploy/compose/env/mirror.env.example @@ -3,8 +3,9 @@ # Core infrastructure credentials MONGO_INITDB_ROOT_USERNAME=stellaops_mirror MONGO_INITDB_ROOT_PASSWORD=mirror-password -MINIO_ROOT_USER=stellaops-mirror -MINIO_ROOT_PASSWORD=mirror-minio-secret +MINIO_ROOT_USER=stellaops-mirror +MINIO_ROOT_PASSWORD=mirror-minio-secret +RUSTFS_HTTP_PORT=8080 # Mirror HTTP listeners MIRROR_GATEWAY_HTTP_PORT=8080 diff --git a/deploy/compose/env/stage.env.example b/deploy/compose/env/stage.env.example index 477e62d1..91157dec 100644 --- a/deploy/compose/env/stage.env.example +++ b/deploy/compose/env/stage.env.example @@ -2,9 +2,10 @@ MONGO_INITDB_ROOT_USERNAME=stellaops MONGO_INITDB_ROOT_PASSWORD=stage-password MINIO_ROOT_USER=stellaops-stage -MINIO_ROOT_PASSWORD=stage-minio-secret -MINIO_CONSOLE_PORT=19001 -AUTHORITY_ISSUER=https://authority.stage.stella-ops.internal +MINIO_ROOT_PASSWORD=stage-minio-secret +MINIO_CONSOLE_PORT=19001 +RUSTFS_HTTP_PORT=8080 +AUTHORITY_ISSUER=https://authority.stage.stella-ops.internal AUTHORITY_PORT=8440 SIGNER_POE_INTROSPECT_URL=https://licensing.stage.stella-ops.internal/introspect SIGNER_PORT=8441 diff --git a/deploy/helm/stellaops/values-airgap.yaml b/deploy/helm/stellaops/values-airgap.yaml index 158c48c5..0241ed8b 100644 --- a/deploy/helm/stellaops/values-airgap.yaml +++ b/deploy/helm/stellaops/values-airgap.yaml @@ -93,15 +93,16 @@ services: volumeClaims: - name: concelier-jobs claimName: stellaops-concelier-jobs - scanner-web: + scanner-web: image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 service: port: 8444 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" SCANNER__EVENTS__ENABLED: "false" SCANNER__EVENTS__DRIVER: "redis" @@ -109,13 +110,14 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - scanner-worker: + scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" SCANNER__EVENTS__ENABLED: "false" SCANNER__EVENTS__DRIVER: "redis" @@ -163,11 +165,11 @@ services: volumeClaims: - name: mongo-data claimName: stellaops-mongo-data - minio: - class: infrastructure - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e - service: - port: 9000 + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 command: - server - /data @@ -179,9 +181,29 @@ services: volumeMounts: - name: minio-data mountPath: /data - volumeClaims: - - name: minio-data - claimName: stellaops-minio-data + volumeClaims: + - name: minio-data + claimName: stellaops-minio-data + rustfs: + class: infrastructure + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + service: + port: 8080 + command: + - serve + - --listen + - 0.0.0.0:8080 + - --root + - /data + env: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumeMounts: + - name: rustfs-data + mountPath: /data + volumeClaims: + - name: rustfs-data + claimName: stellaops-rustfs-data nats: class: infrastructure image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e diff --git a/deploy/helm/stellaops/values-dev.yaml b/deploy/helm/stellaops/values-dev.yaml index 2f13c090..a27d75d3 100644 --- a/deploy/helm/stellaops/values-dev.yaml +++ b/deploy/helm/stellaops/values-dev.yaml @@ -51,7 +51,7 @@ configMaps: telemetry: enableRequestLogging: true minimumLogLevel: Debug -services: +services: authority: image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd service: @@ -92,15 +92,16 @@ services: volumes: - name: concelier-jobs emptyDir: {} - scanner-web: + scanner-web: image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 service: port: 8444 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" SCANNER__EVENTS__ENABLED: "false" SCANNER__EVENTS__DRIVER: "redis" @@ -108,13 +109,14 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - scanner-worker: + scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" SCANNER__EVENTS__ENABLED: "false" SCANNER__EVENTS__DRIVER: "redis" @@ -161,9 +163,9 @@ services: volumes: - name: mongo-data emptyDir: {} - minio: - class: infrastructure - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e service: port: 9000 command: @@ -177,9 +179,23 @@ services: volumeMounts: - name: minio-data mountPath: /data - volumes: - - name: minio-data - emptyDir: {} + volumes: + - name: minio-data + emptyDir: {} + rustfs: + class: infrastructure + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + service: + port: 8080 + env: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumeMounts: + - name: rustfs-data + mountPath: /data + volumes: + - name: rustfs-data + emptyDir: {} nats: class: infrastructure image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e diff --git a/deploy/helm/stellaops/values-stage.yaml b/deploy/helm/stellaops/values-stage.yaml index 8777e0be..976c256f 100644 --- a/deploy/helm/stellaops/values-stage.yaml +++ b/deploy/helm/stellaops/values-stage.yaml @@ -92,15 +92,16 @@ services: volumeClaims: - name: concelier-jobs claimName: stellaops-concelier-jobs - scanner-web: + scanner-web: image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 service: port: 8444 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" SCANNER__EVENTS__ENABLED: "false" SCANNER__EVENTS__DRIVER: "redis" @@ -108,14 +109,15 @@ services: SCANNER__EVENTS__STREAM: "stella.events" SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5" SCANNER__EVENTS__MAXSTREAMLENGTH: "10000" - scanner-worker: + scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab replicas: 2 env: SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" - SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" - SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" - SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + SCANNER__ARTIFACTSTORE__DRIVER: "rustfs" + SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1" + SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts" + SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30" SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" SCANNER__EVENTS__ENABLED: "false" SCANNER__EVENTS__DRIVER: "redis" @@ -162,11 +164,11 @@ services: volumeClaims: - name: mongo-data claimName: stellaops-mongo-data - minio: - class: infrastructure - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e - service: - port: 9000 + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 command: - server - /data @@ -178,9 +180,29 @@ services: volumeMounts: - name: minio-data mountPath: /data - volumeClaims: - - name: minio-data - claimName: stellaops-minio-data + volumeClaims: + - name: minio-data + claimName: stellaops-minio-data + rustfs: + class: infrastructure + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + service: + port: 8080 + command: + - serve + - --listen + - 0.0.0.0:8080 + - --root + - /data + env: + RUSTFS__LOG__LEVEL: info + RUSTFS__STORAGE__PATH: /data + volumeMounts: + - name: rustfs-data + mountPath: /data + volumeClaims: + - name: rustfs-data + claimName: stellaops-rustfs-data nats: class: infrastructure image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e diff --git a/deploy/releases/2025.10-edge.yaml b/deploy/releases/2025.10-edge.yaml index 6c7e75df..e2f65881 100644 --- a/deploy/releases/2025.10-edge.yaml +++ b/deploy/releases/2025.10-edge.yaml @@ -1,8 +1,8 @@ -release: - version: "2025.10.0-edge" - channel: "edge" - date: "2025-10-01T00:00:00Z" - calendar: "2025.10" + release: + version: "2025.10.0-edge" + channel: "edge" + date: "2025-10-01T00:00:00Z" + calendar: "2025.10" components: - name: authority image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd @@ -20,10 +20,12 @@ release: image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 - name: web-ui image: registry.stella-ops.org/stellaops/web-ui@sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf - infrastructure: - mongo: - image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 - minio: - image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e - checksums: - releaseManifestSha256: 822f82987529ea38d2321dbdd2ef6874a4062a117116a20861c26a8df1807beb + infrastructure: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + rustfs: + image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge + checksums: + releaseManifestSha256: 64d5b05c864bbfaeb29dad3958f4e7ff43d13393059da558ab355cebb9aba2b7 diff --git a/docs/07_HIGH_LEVEL_ARCHITECTURE.md b/docs/07_HIGH_LEVEL_ARCHITECTURE.md index 3a11036b..9847259a 100755 --- a/docs/07_HIGH_LEVEL_ARCHITECTURE.md +++ b/docs/07_HIGH_LEVEL_ARCHITECTURE.md @@ -49,7 +49,7 @@ * **Fulcio** (Sigstore CA) — issues short‑lived signing certs (keyless). * **Rekor v2** (tile‑backed transparency log). -* **MinIO** — S3‑compatible object store with lifecycle & Object Lock. +* **RustFS** — offline-first object store with deterministic REST API (S3/MinIO fallback available for legacy installs). * **MongoDB** — catalog, advisories, VEX, scheduler, notify. * **Queue** — Redis Streams / NATS / RabbitMQ (pluggable). * **OCI Registry** — must support **Referrers API** (discover SBOMs/signatures). @@ -81,7 +81,7 @@ flowchart LR ATT[Attestor\n(Rekor v2 submit/verify)] UI[Web UI (Angular)] Z[Zastava\n(Runtime Inspector/Enforcer)] - MIN[(MinIO S3)] + RFS[(RustFS object store)] MGO[(MongoDB)] QUE[(Queue/Streams)] end @@ -94,7 +94,7 @@ flowchart LR CLI -->|scan/build| SW SW -->|jobs| QUE QUE --> WK - WK --> MIN + WK --> RFS SW --> MGO CONC --> MGO EXC --> MGO @@ -225,13 +225,13 @@ LS --> IA: PoE (mTLS client cert or JWT with cnf=K_inst), CRL/OCSP/introspect --- -## 6) Storage & catalogs (MinIO/Mongo) +## 6) Storage & catalogs (RustFS/Mongo) + +**RustFS layout (default)** -**MinIO layout** - -``` -s3://stellaops/ - layers//sbom.cdx.json.zst +``` +rustfs://stellaops/ + layers//sbom.cdx.json.zst layers//sbom.spdx.json.zst images//inventory.cdx.pb images//usage.cdx.pb @@ -248,7 +248,7 @@ s3://stellaops/ **Retention** -* MinIO **ILM** for coarse TTL; Scanner.WebService GC decrements `refCount` and deletes unreferenced metadata; **Object Lock** for immutable classes (auditable artifacts). +* RustFS applies retention via `X-RustFS-Retain-Seconds`; Scanner.WebService GC decrements `refCount` and deletes unreferenced metadata; S3/MinIO fallback retains native Object Lock when enabled. --- @@ -395,7 +395,7 @@ services: ui: { image: stellaops/ui, depends_on: [scanner-web, concelier, excititor, scheduler-web, notify-web] } ``` -* **Backups:** Mongo dumps; MinIO versioned buckets & replication; Rekor v2 DB snapshots; JWKS/Fulcio/KMS key rotation. +* **Backups:** Mongo dumps; RustFS snapshots (or S3 versioning when fallback driver is used); Rekor v2 DB snapshots; JWKS/Fulcio/KMS key rotation. * **Ops runbooks:** Scheduler catch‑up after Concelier/Excititor recovery; connector key rotation (Slack/Teams/SMTP). * **SLOs & alerts:** lag between Concelier/Excititor export and first rescan verdict; delivery failure rates by channel. @@ -408,7 +408,7 @@ services: * **Notify metrics:** `notify.sent_total{channel}`, `notify.dropped_total{reason}`, `notify.digest_coalesced_total`, `notify.latency_ms`. * **Tracing:** per‑stage spans; correlation IDs across Scanner→Signer→Attestor and Concelier/Excititor→Scheduler→Scanner→Notify. * **Audit logs:** every signing records `license_id`, `image_digest`, `policy_digest`, and Rekor UUID; Scheduler records who scheduled what; Notify records where, when, and why messages were sent or deduped. -* **Compliance:** MinIO **Object Lock** for immutable artifacts; reproducible outputs via policy digest + SBOM digest in predicate. +* **Compliance:** RustFS retention headers (or MinIO Object Lock when operating in S3 mode) keep immutable artifacts tamper‑resistant; reproducible outputs via policy digest + SBOM digest in predicate. --- diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index c08069f7..950b43b2 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -382,28 +382,65 @@ Request body mirrors policy preview inputs (image digest plus findings). The ser ```json { "report": { - "reportId": "report-3def5f362aa475ef14b6", - "imageDigest": "sha256:deadbeef", + "reportId": "report-9f8cde21aab54321", + "imageDigest": "sha256:7dbe0c9a5d4f1c8184007e9d94dbe55928f8a2db5ab9c1c2d4a2f7bbcdfe1234", + "generatedAt": "2025-10-23T15:32:22Z", "verdict": "blocked", - "policy": { "revisionId": "rev-1", "digest": "27d2ec2b34feedc304fc564d252ecee1c8fa14ea581a5ff5c1ea8963313d5c8d" }, - "summary": { "total": 1, "blocked": 1, "warned": 0, "ignored": 0, "quieted": 0 }, + "policy": { + "revisionId": "rev-42", + "digest": "8a0f72f8dc5c51c46991db3bba34e9b3c0c8e944a7a6d0a9c29a9aa6b8439876" + }, + "summary": { "total": 2, "blocked": 1, "warned": 1, "ignored": 0, "quieted": 0 }, "verdicts": [ { - "findingId": "finding-1", + "findingId": "library:pkg/openssl@1.1.1w", "status": "Blocked", - "ruleName": "Block Critical", - "ruleAction": "Block", - "score": 40.5, + "ruleName": "Block vendor unknowns", + "ruleAction": "block", + "notes": "Unknown vendor telemetry — medium confidence band.", + "score": 19.5, "configVersion": "1.0", "inputs": { - "reachabilityWeight": 0.45, - "baseScore": 40.5, - "severityWeight": 90, - "trustWeight": 1, - "trustWeight.NVD": 1, - "reachability.runtime": 0.45 + "severityWeight": 50, + "trustWeight": 0.65, + "reachabilityWeight": 0.6, + "baseScore": 19.5, + "trustWeight.vendor": 0.65, + "reachability.unknown": 0.6, + "unknownConfidence": 0.55, + "unknownAgeDays": 5 }, + "quietedBy": null, "quiet": false, + "unknownConfidence": 0.55, + "confidenceBand": "medium", + "unknownAgeDays": 5, + "sourceTrust": "vendor", + "reachability": "unknown" + }, + { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Warned", + "ruleName": "Runtime mitigation required", + "ruleAction": "warn", + "notes": "Runtime reachable unknown — mitigation window required.", + "score": 18.75, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75, + "reachability.runtime": 0.45, + "warnPenalty": 15, + "unknownConfidence": 0.35, + "unknownAgeDays": 13 + }, + "quietedBy": null, + "quiet": false, + "unknownConfidence": 0.35, + "confidenceBand": "medium", + "unknownAgeDays": 13, "sourceTrust": "NVD", "reachability": "runtime" } @@ -412,21 +449,21 @@ Request body mirrors policy preview inputs (image digest plus findings). The ser }, "dsse": { "payloadType": "application/vnd.stellaops.report+json", - "payload": "", + "payload": "eyJyZXBvcnQiOnsicmVwb3J0SWQiOiJyZXBvcnQtOWY4Y2RlMjFhYWI1NDMyMSJ9fQ==", "signatures": [ { "keyId": "scanner-report-signing", "algorithm": "hs256", - "signature": "" + "signature": "MEQCIGHscnJ2bm9wYXlsb2FkZXIAIjANBgkqhkiG9w0BAQsFAAOCAQEASmFja3Nvbk1ldGE=" } ] } } ``` -- The `report` object omits null fields and is deterministic (ISO timestamps, sorted keys). +- The `report` object omits null fields and is deterministic (ISO timestamps, sorted keys) while surfacing `unknownConfidence`, `confidenceBand`, and `unknownAgeDays` for auditability. - `dsse` follows the DSSE (Dead Simple Signing Envelope) shape; `payload` is the canonical UTF-8 JSON and `signatures[0].signature` is the base64 HMAC/Ed25519 value depending on configuration. -- A runnable sample envelope is available at `samples/api/reports/report-sample.dsse.json` for tooling tests or signature verification. +- Full offline samples live at `samples/policy/policy-report-unknown.json` (request + response) and `samples/api/reports/report-sample.dsse.json` (envelope fixture) for tooling tests or signature verification. **Response 404** – `application/problem+json` payload with type `https://stellaops.org/problems/not-found` when the scan identifier is unknown. diff --git a/docs/11_AUTHORITY.md b/docs/11_AUTHORITY.md index 3bc03df2..93b77d22 100644 --- a/docs/11_AUTHORITY.md +++ b/docs/11_AUTHORITY.md @@ -176,8 +176,9 @@ Authority now understands two flavours of sender-constrained OAuth clients: ``` Operators can override any field via environment variables (e.g. `STELLAOPS_AUTHORITY__SECURITY__SENDERCONSTRAINTS__DPOP__NONCE__STORE=redis`). - Declare client `audiences` in bootstrap manifests or plug-in provisioning metadata; Authority now defaults the token `aud` claim and `resource` indicator from this list, which is also used to trigger nonce enforcement for audiences such as `signer` and `attestor`. -- **Mutual TLS clients** – client registrations may declare an mTLS binding (`senderConstraint: mtls`). When enabled via `security.senderConstraints.mtls`, Authority validates the presented client certificate against stored bindings (`certificateBindings[]`), optional chain verification, and timing windows. Successful requests embed `cnf.x5t#S256` into the access token so resource servers can enforce the certificate thumbprint. - - Certificate bindings record the certificate thumbprint, optional SANs, subject/issuer metadata, and activation windows. Operators can enforce subject regexes, SAN type allow-lists (`dns`, `uri`, `ip`), trusted certificate authorities, and rotation grace via `security.senderConstraints.mtls.*`. +- **Mutual TLS clients** – client registrations may declare an mTLS binding (`senderConstraint: mtls`). When enabled via `security.senderConstraints.mtls`, Authority validates the presented client certificate against stored bindings (`certificateBindings[]`), optional chain verification, and timing windows. Successful requests embed `cnf.x5t#S256` into the access token (and introspection output) so resource servers can enforce the certificate thumbprint. + - `security.senderConstraints.mtls.enforceForAudiences` forces mTLS whenever the requested `aud`/`resource` (or the client's configured audiences) intersect the configured allow-list (default includes `signer`). Clients configured for different sender constraints are rejected early so operator policy remains consistent. + - Certificate bindings now act as an allow-list: Authority verifies thumbprint, subject, issuer, serial number, and any declared SAN values against the presented certificate, with rotation grace windows applied to `notBefore/notAfter`. Operators can enforce subject regexes, SAN type allow-lists (`dns`, `uri`, `ip`), trusted certificate authorities, and rotation grace via `security.senderConstraints.mtls.*`. Both modes persist additional metadata in `authority_tokens`: `senderConstraint` records the enforced policy, while `senderKeyThumbprint` stores the DPoP JWK thumbprint or mTLS certificate hash captured at issuance. Downstream services can rely on these fields (and the corresponding `cnf` claim) when auditing offline copies of the token store. diff --git a/docs/11_DATA_SCHEMAS.md b/docs/11_DATA_SCHEMAS.md index c110a897..e99c1654 100755 --- a/docs/11_DATA_SCHEMAS.md +++ b/docs/11_DATA_SCHEMAS.md @@ -306,7 +306,22 @@ Validation occurs alongside policy binding (`PolicyScoringConfigBinder`), produc **Runtime usage** - `trustOverrides` are matched against `finding.tags` (`trust:`) first, then `finding.source`/`finding.vendor`; missing keys default to `1.0`. - `reachabilityBuckets` consume `finding.tags` with prefix `reachability:` (fallback `usage:` or `unknown`). Missing buckets fall back to `unknown` weight when present, otherwise `1.0`. -- Policy verdicts expose scoring inputs (`severityWeight`, `trustWeight`, `reachabilityWeight`, `baseScore`, penalties) plus unknown-state metadata (`unknownConfidence`, `unknownAgeDays`, `confidenceBand`) for auditability. See `samples/policy/policy-preview-unknown.json` for an end-to-end preview payload. +- Policy verdicts expose scoring inputs (`severityWeight`, `trustWeight`, `reachabilityWeight`, `baseScore`, penalties) plus unknown-state metadata (`unknownConfidence`, `unknownAgeDays`, `confidenceBand`) for auditability. See `samples/policy/policy-preview-unknown.json` and `samples/policy/policy-report-unknown.json` for offline reference payloads validated against the published schemas below. + +Validate the samples locally with **Ajv** before publishing changes: + +```bash +# install once per checkout (offline-safe): +npm install --no-save ajv-cli@5 ajv-formats@2 + +npx ajv validate --spec=draft2020 -c ajv-formats \ + -s docs/schemas/policy-preview-sample@1.json \ + -d samples/policy/policy-preview-unknown.json + +npx ajv validate --spec=draft2020 -c ajv-formats \ + -s docs/schemas/policy-report-sample@1.json \ + -d samples/policy/policy-report-unknown.json +``` - Unknown confidence derives from `unknown-age-days:` (preferred) or `unknown-since:` + `observed-at:` tags; with no hints the engine keeps `initial` confidence. Values decay by `decayPerDay` down to `floor`, then resolve to the first matching `bands[].name`. --- diff --git a/docs/12_PERFORMANCE_WORKBOOK.md b/docs/12_PERFORMANCE_WORKBOOK.md index 9fdb9355..fcfa7ce9 100755 --- a/docs/12_PERFORMANCE_WORKBOOK.md +++ b/docs/12_PERFORMANCE_WORKBOOK.md @@ -56,7 +56,7 @@ ## 3 Test Harness * **Runner** – `perf/run.sh`, accepts `--phase` and `--samples`. -* **Language analyzers microbench** – `dotnet run --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj -- --repo-root . --out bench/Scanner.Analyzers/baseline.csv` produces deterministic CSVs for analyzer scenarios (Node today, others as they land). +* **Language analyzers microbench** – `dotnet run --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj -- --repo-root . --out bench/Scanner.Analyzers/baseline.csv --json out/bench/scanner-analyzers/latest.json --prom out/bench/scanner-analyzers/latest.prom --commit $(git rev-parse HEAD)` produces CSV + JSON + Prometheus gauges for analyzer scenarios. Runs fail if `max_ms` regresses ≥ 20 % against `baseline.csv` or if thresholds are exceeded. * **Metrics** – Prometheus + `jq` extracts; aggregated via `scripts/aggregate.ts`. * **CI** – GitLab CI job *benchmark* publishes JSON to `bench‑artifacts/`. * **Visualisation** – Grafana dashboard *Stella‑Perf* (provisioned JSON). @@ -143,7 +143,9 @@ P99 = 48 ms. Meets 50 ms gate. ## 8 Trend Snapshot -![Perf trend spark‑line placeholder](perf‑trend.svg) +![Perf trend spark‑line placeholder](perf‑trend.svg) + +> **Grafana/Alerting** – Import `docs/ops/scanner-analyzers-grafana-dashboard.json` and point it at the Prometheus datasource storing `scanner_analyzer_bench_*` metrics. Configure an alert on `scanner_analyzer_bench_regression_ratio` ≥ 1.20 (default limit); the bundled Stat panel surfaces breached scenarios (non-zero values). On-call runbook: `docs/ops/scanner-analyzers-operations.md`. _Plot generated weekly by `scripts/update‑trend.py`; shows last 12 weeks P95 per phase._ diff --git a/docs/15_UI_GUIDE.md b/docs/15_UI_GUIDE.md index 5537246c..77051f2d 100755 --- a/docs/15_UI_GUIDE.md +++ b/docs/15_UI_GUIDE.md @@ -96,11 +96,29 @@ _No external fonts or JS – true offline guarantee._ | **Import / Export** | Buttons map to `/policy/import` and `/policy/export`. Accepts `.yaml`, `.rego`, `.zip` (bundle). | | **History** | Immutable audit log; diff viewer highlights rule changes. | -#### 3.3.1 YAML → Rego Bridge - -If you paste YAML but enable **Strict Mode** (toggle), backend converts to Rego under the hood, stores both representations, and shows a side‑by‑side diff. - -### 3.4 📌 Settings Enhancements +#### 3.3.1 YAML → Rego Bridge + +If you paste YAML but enable **Strict Mode** (toggle), backend converts to Rego under the hood, stores both representations, and shows a side‑by‑side diff. + +#### 3.3.2 Preview / Report Fixtures + +- Use the offline fixtures (`samples/policy/policy-preview-unknown.json` and `samples/policy/policy-report-unknown.json`) to exercise the Policies screens without a live backend; both payloads include confidence bands, unknown-age tags, and scoring inputs that map directly to the UI panels. +- Keep them in lock-step with the API by validating any edits with Ajv: + + ```bash + # install once per checkout (offline-safe): + npm install --no-save ajv-cli@5 ajv-formats@2 + + npx ajv validate --spec=draft2020 -c ajv-formats \ + -s docs/schemas/policy-preview-sample@1.json \ + -d samples/policy/policy-preview-unknown.json + + npx ajv validate --spec=draft2020 -c ajv-formats \ + -s docs/schemas/policy-report-sample@1.json \ + -d samples/policy/policy-report-unknown.json + ``` + +### 3.4 📌 Settings Enhancements | Setting | Details | | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md index 974c6e97..bbe6a42f 100755 --- a/docs/24_OFFLINE_KIT.md +++ b/docs/24_OFFLINE_KIT.md @@ -17,11 +17,11 @@ completely isolated network: | **Provenance** | Cosign signature, SPDX 2.3 SBOM, in‑toto SLSA attestation | | **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | | **Delta patches** | Daily diff bundles keep size \< 350 MB | -| **Scanner plug-ins** | OS analyzers plus the Node.js, Go, and .NET language analyzers packaged under `plugins/scanner/analyzers/**` with manifests so Workers load deterministically offline. | +| **Scanner plug-ins** | OS analyzers plus the Node.js, Go, .NET, and Python language analyzers packaged under `plugins/scanner/analyzers/**` with manifests so Workers load deterministically offline. | **RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an air‑gapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache. -**Language analyzers:** the kit now carries the restart-only Node.js, Go, and .NET analyzer plug-ins (`plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Node/`, `plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Go/`, `plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.DotNet/`). Drop the directories alongside Worker binaries so the unified plug-in catalog can load them without outbound fetches; upcoming Python/Rust plug-ins will follow the same layout. +**Language analyzers:** the kit now carries the restart-only Node.js, Go, .NET, and Python analyzer plug-ins (`plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Node/`, `...Lang.Go/`, `...Lang.DotNet/`, `...Lang.Python/`). Drop the directories alongside Worker binaries so the unified plug-in catalog can load them without outbound fetches; Rust remains on the Wave 4 roadmap. *Scanner core:* C# 12 on **.NET {{ dotnet }}**. *Imports are idempotent and atomic — no service downtime.* @@ -99,6 +99,24 @@ Example excerpt (2025-10-23 kit) showing the Go and .NET analyzer plug-in payloa "size": 647, "capturedAt": "2025-10-23T00:00:00Z" } +{ + "name": "plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.dll", + "sha256": "28b6e06c7cabf3b78f13f801cbb14962093f3d42c4ae9ec01babbcd14cda4644", + "size": 53760, + "capturedAt": "2025-10-23T00:00:00Z" +} +{ + "name": "plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.pdb", + "sha256": "be4e34b4dc9a790fe1299e84213343b7c8ea90a2d22e5d7d1aa7585b8fedc946", + "size": 34516, + "capturedAt": "2025-10-23T00:00:00Z" +} +{ + "name": "plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/manifest.json", + "sha256": "bceea1e7542aae860b0ec5ba7b8b3aa960b21edc4d1efe60afc98ce289341ac3", + "size": 671, + "capturedAt": "2025-10-23T00:00:00Z" +} ``` --- @@ -130,7 +148,7 @@ The CLI validates recorded digests (when `.metadata.json` is present) before str **Quick smoke test:** before import, verify the tarball carries the Go analyzer plug-in: ```bash -tar -tzf stella-ops-offline-kit-.tgz 'plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Go/*' 'plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.DotNet/*' +tar -tzf stella-ops-offline-kit-.tgz 'plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Go/*' 'plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.DotNet/*' 'plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Python/*' ``` The manifest lookup above and this `tar` listing should both surface the Go analyzer DLL, PDB, and manifest entries before the kit is promoted. diff --git a/docs/ARCHITECTURE_AUTHORITY.md b/docs/ARCHITECTURE_AUTHORITY.md index bfa5f8d0..edccfdb8 100644 --- a/docs/ARCHITECTURE_AUTHORITY.md +++ b/docs/ARCHITECTURE_AUTHORITY.md @@ -99,6 +99,8 @@ plan? = // optional hint for UIs; not used for e * **Client Credentials** (service→service): * **mTLS**: mutual TLS + `client_id` → bound token (`cnf.x5t#S256`) + * `security.senderConstraints.mtls.enforceForAudiences` forces the mTLS path when requested `aud`/`resource` values intersect high-value audiences (defaults include `signer`). Authority rejects clients attempting to use DPoP/basic secrets for these audiences. + * Stored `certificateBindings` are authoritative: thumbprint, subject, issuer, serial number, and SAN values are matched against the presented certificate, with rotation grace applied to activation windows. Failures surface deterministic error codes (e.g. `certificate_binding_subject_mismatch`). * **private_key_jwt**: JWT‑based client auth + **DPoP** header (preferred for tools and CLI) * **Device Code** (CLI): `POST /oauth/device/code` + `POST /oauth/token` poll * **Authorization Code + PKCE** (UI): standard diff --git a/docs/ARCHITECTURE_DEVOPS.md b/docs/ARCHITECTURE_DEVOPS.md index d4d89df9..919de2c1 100644 --- a/docs/ARCHITECTURE_DEVOPS.md +++ b/docs/ARCHITECTURE_DEVOPS.md @@ -1,6 +1,6 @@ # component_architecture_devops.md — **Stella Ops Release & Operations** (2025Q4) -> **Scope.** Implementation‑ready blueprint for **how Stella Ops is built, versioned, signed, distributed, upgraded, licensed (PoE)**, and operated in customer environments (online and air‑gapped). Covers reproducible builds, supply‑chain attestations, registries, offline kits, migration/rollback, artifact lifecycle (MinIO/Mongo), monitoring SLOs, and customer activation. +> **Scope.** Implementation‑ready blueprint for **how Stella Ops is built, versioned, signed, distributed, upgraded, licensed (PoE)**, and operated in customer environments (online and air‑gapped). Covers reproducible builds, supply‑chain attestations, registries, offline kits, migration/rollback, artifact lifecycle (RustFS default + Mongo, S3 fallback), monitoring SLOs, and customer activation. --- @@ -257,12 +257,12 @@ Signer validates **scanner** image’s cosign identity + calendar tag for **rele --- -## 7) Artifact lifecycle & storage (MinIO/Mongo) +## 7) Artifact lifecycle & storage (RustFS/Mongo) -### 7.1 Buckets & prefixes (MinIO) +### 7.1 Buckets & prefixes (RustFS) ``` -s3://stellaops/ +rustfs://stellaops/ scanner/ layers//sbom.cdx.json.zst images//inventory.cdx.pb @@ -283,7 +283,7 @@ s3://stellaops/ * **`short`**: working artifacts (diffs, queues) — TTL 7–14 days. * **`default`**: SBOMs & indexes — TTL 90–180 days (configurable). -* **`compliance`**: signed reports & attested exports — **Object Lock** (governance/compliance) 1–7 years. +* **`compliance`**: signed reports & attested exports — retention enforced via RustFS hold or S3 Object Lock (governance/compliance) 1–7 years. ### 7.3 Artifact Lifecycle Controller (ALC) @@ -292,6 +292,9 @@ s3://stellaops/ * Artifacts referenced by **reports** or **tickets** are pinned. * ILM actions logged; UI shows per‑class usage & upcoming purges. +> **Migration note.** Follow `docs/ops/scanner-rustfs-migration.md` when transitioning existing +> MinIO buckets to RustFS. The provided migrator is idempotent and safe to rerun per prefix. + ### 7.4 Mongo retention * **Scanner**: `runtime.events` use TTL (e.g., 30–90 days); **catalog** permanent. @@ -313,7 +316,7 @@ s3://stellaops/ * **Golden signals**: * **Latency**: token issuance, sign→attest round‑trip, scan enqueue→emit, export build. - * **Saturation**: queue depth, Mongo write IOPS, MinIO net throughput. + * **Saturation**: queue depth, Mongo write IOPS, RustFS throughput / queue depth (or S3 metrics when in fallback mode). * **Traffic**: scans/min, attestations/min, webhook admits/min. * **Errors**: 5xx rates, cosign verification failures, Rekor timeouts. @@ -460,7 +463,7 @@ services: * `attestor.submit_latency_seconds{quantile=0.95}` < 0.3. * `scanner.scan_latency_seconds{quantile=0.95}` < target per image size. * `concelier.export.duration_seconds` stable; `excititor.consensus.conflicts_total` not exploding after policy changes. -* MinIO `s3_requests_errors_total` near zero; Mongo `opcounters` hit expected baseline. +* RustFS request error rate near zero (or `s3_requests_errors_total` when operating against S3); Mongo `opcounters` hit expected baseline. ### Appendix B — Upgrade safety checklist diff --git a/docs/ARCHITECTURE_SCANNER.md b/docs/ARCHITECTURE_SCANNER.md index 19fa34ee..d1202068 100644 --- a/docs/ARCHITECTURE_SCANNER.md +++ b/docs/ARCHITECTURE_SCANNER.md @@ -1,6 +1,6 @@ # component_architecture_scanner.md — **Stella Ops Scanner** (2025Q4) -> **Scope.** Implementation‑ready architecture for the **Scanner** subsystem: WebService, Workers, analyzers, SBOM assembly (inventory & usage), per‑layer caching, three‑way diffs, artifact catalog (MinIO+Mongo), attestation hand‑off, and scale/security posture. This document is the contract between the scanning plane and everything else (Policy, Excititor, Concelier, UI, CLI). +> **Scope.** Implementation‑ready architecture for the **Scanner** subsystem: WebService, Workers, analyzers, SBOM assembly (inventory & usage), per‑layer caching, three‑way diffs, artifact catalog (RustFS default + Mongo, S3-compatible fallback), attestation hand‑off, and scale/security posture. This document is the contract between the scanning plane and everything else (Policy, Excititor, Concelier, UI, CLI). --- @@ -23,7 +23,7 @@ src/ ├─ StellaOps.Scanner.WebService/ # REST control plane, catalog, diff, exports ├─ StellaOps.Scanner.Worker/ # queue consumer; executes analyzers ├─ StellaOps.Scanner.Models/ # DTOs, evidence, graph nodes, CDX/SPDX adapters - ├─ StellaOps.Scanner.Storage/ # Mongo repositories; MinIO object client; ILM/GC + ├─ StellaOps.Scanner.Storage/ # Mongo repositories; RustFS object client (default) + S3 fallback; ILM/GC ├─ StellaOps.Scanner.Queue/ # queue abstraction (Redis/NATS/RabbitMQ) ├─ StellaOps.Scanner.Cache/ # layer cache; file CAS; bloom/bitmap indexes ├─ StellaOps.Scanner.EntryTrace/ # ENTRYPOINT/CMD → terminal program resolver (shell AST) @@ -81,7 +81,7 @@ The DI extension (`AddScannerQueue`) wires the selected transport, so future add ## 2) External dependencies * **OCI registry** with **Referrers API** (discover attached SBOMs/signatures). -* **MinIO** (S3‑compatible) for SBOM artifacts; **Object Lock** for immutable classes; **ILM** for TTL. +* **RustFS** (default, offline-first) for SBOM artifacts; optional S3/MinIO compatibility retained for migration; **Object Lock** semantics emulated via retention headers; **ILM** for TTL. * **MongoDB** for catalog, job state, diffs, ILM rules. * **Queue** (Redis Streams/NATS/RabbitMQ). * **Authority** (on‑prem OIDC) for **OpToks** (DPoP/mTLS). @@ -133,7 +133,7 @@ No confidences. Either a fact is proven with listed mechanisms, or it is not cla * `jobs { _id, kind, args, state, startedAt, heartbeatAt, endedAt, error }` * `lifecycleRules { ruleId, scope, ttlDays, retainIfReferenced, immutable }` -### 3.3 Object store layout (MinIO) +### 3.3 Object store layout (RustFS) ``` layers//sbom.cdx.json.zst @@ -145,6 +145,13 @@ diffs/_/diff.json.zst attest/.dsse.json # DSSE bundle (cert chain + Rekor proof) ``` +RustFS exposes a deterministic HTTP API (`PUT|GET|DELETE /api/v1/buckets/{bucket}/objects/{key}`). +Scanner clients tag immutable uploads with `X-RustFS-Immutable: true` and, when retention applies, +`X-RustFS-Retain-Seconds: `. Additional headers can be injected via +`scanner.artifactStore.headers` to support custom auth or proxy requirements. Legacy MinIO/S3 +deployments remain supported by setting `scanner.artifactStore.driver = "s3"` during phased +migrations. + --- ## 4) REST API (Scanner.WebService) @@ -396,7 +403,7 @@ scanner: * **HA**: WebService horizontal scale; Workers autoscale by queue depth & CPU; distributed locks on layers. * **Retention**: ILM rules per artifact class (`short`, `default`, `compliance`); **Object Lock** for compliance artifacts (reports, signed SBOMs). * **Upgrades**: bump **cache schema** when analyzer outputs change; WebService triggers refresh of dependent artifacts. -* **Backups**: Mongo (daily dumps); MinIO (versioned buckets, replication); Rekor v2 DB snapshots. +* **Backups**: Mongo (daily dumps); RustFS snapshots (filesystem-level rsync/ZFS) or S3 versioning when legacy driver enabled; Rekor v2 DB snapshots. --- diff --git a/docs/README.md b/docs/README.md index 43c91e5d..f16d683e 100755 --- a/docs/README.md +++ b/docs/README.md @@ -80,6 +80,8 @@ Everything here is open‑source and versioned — when you check out a git ta - **29 – [Concelier CISA ICS Connector Operations](ops/concelier-icscisa-operations.md)** - **30 – [Concelier CERT-Bund Connector Operations](ops/concelier-certbund-operations.md)** - **31 – [Concelier MSRC Connector – AAD Onboarding](ops/concelier-msrc-operations.md)** +- **32 – [Scanner Analyzer Bench Operations](ops/scanner-analyzers-operations.md)** +- **33 – [Scanner Artifact Store Migration](ops/scanner-rustfs-migration.md)** ### Legal & licence - **32 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** diff --git a/docs/dev/authority-dpop-mtls-plan.md b/docs/dev/authority-dpop-mtls-plan.md index f454c25f..7cc6e792 100644 --- a/docs/dev/authority-dpop-mtls-plan.md +++ b/docs/dev/authority-dpop-mtls-plan.md @@ -14,6 +14,8 @@ > **Status update (2025-10-19):** `ValidateDpopProofHandler`, `AuthorityClientCertificateValidator`, and the supporting storage/audit plumbing now live in `src/StellaOps.Authority`. DPoP proofs populate `cnf.jkt`, mTLS bindings enforce certificate thumbprints via `cnf.x5t#S256`, and token documents persist the sender constraint metadata. In-memory nonce issuance is wired (Redis implementation to follow). Documentation and configuration references were updated (`docs/11_AUTHORITY.md`). Targeted unit/integration tests were added; running the broader test suite is currently blocked by pre-existing `StellaOps.Concelier.Storage.Mongo` build errors. > > **Status update (2025-10-20):** Redis-backed nonce configuration is exposed through `security.senderConstraints.dpop.nonce` with sample YAML (`etc/authority.yaml.sample`) and architecture docs refreshed. Operator guide now includes concrete Redis/required audiences snippet; nonce challenge regression remains covered by `ValidateDpopProof_IssuesNonceChallenge_WhenNonceMissing`. +> +> **Status update (2025-10-23):** mTLS enforcement now honours `security.senderConstraints.mtls.enforceForAudiences`, automatically rejecting non-mTLS clients targeting audiences such as `signer`. Certificate bindings validate thumbprint, issuer, subject, serial number, and SAN values, producing deterministic error codes for operators. Introspection responses include `cnf.x5t#S256`, and new unit tests cover audience enforcement, binding mismatches, and bootstrap storage. Docs/sample config updated accordingly. ## Design Summary - Extract the existing Scanner `DpopProofValidator` stack into a shared `StellaOps.Auth.Security` library used by Authority and resource servers. diff --git a/docs/ops/scanner-analyzers-grafana-dashboard.json b/docs/ops/scanner-analyzers-grafana-dashboard.json new file mode 100644 index 00000000..0882a568 --- /dev/null +++ b/docs/ops/scanner-analyzers-grafana-dashboard.json @@ -0,0 +1,155 @@ +{ + "title": "StellaOps Scanner Analyzer Benchmarks", + "uid": "scanner-analyzer-bench", + "schemaVersion": 38, + "version": 1, + "editable": true, + "timezone": "", + "graphTooltip": 0, + "time": { + "from": "now-24h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "hide": 0, + "current": {} + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "id": 1, + "title": "Max Duration (ms)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "displayName": "{{scenario}}" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "scanner_analyzer_bench_max_ms", + "legendFormat": "{{scenario}}", + "refId": "A" + }, + { + "expr": "scanner_analyzer_bench_baseline_max_ms", + "legendFormat": "{{scenario}} baseline", + "refId": "B" + } + ] + }, + { + "id": 2, + "title": "Regression Ratio vs Limit", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "displayName": "{{scenario}}", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "(scanner_analyzer_bench_regression_ratio - 1) * 100", + "legendFormat": "{{scenario}} regression %", + "refId": "A" + }, + { + "expr": "(scanner_analyzer_bench_regression_limit - 1) * 100", + "legendFormat": "{{scenario}} limit %", + "refId": "B" + } + ] + }, + { + "id": 3, + "title": "Breached Scenarios", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "displayName": "{{scenario}}", + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "expr": "scanner_analyzer_bench_regression_breached", + "legendFormat": "{{scenario}}", + "refId": "A" + } + ] + } + ] +} diff --git a/docs/ops/scanner-analyzers-operations.md b/docs/ops/scanner-analyzers-operations.md new file mode 100644 index 00000000..a719f7ff --- /dev/null +++ b/docs/ops/scanner-analyzers-operations.md @@ -0,0 +1,48 @@ +# Scanner Analyzer Benchmarks – Operations Guide + +## Purpose +Keep the language analyzer microbench under the < 5 s SBOM pledge. CI emits Prometheus metrics and JSON fixtures so trend dashboards and alerts stay in lockstep with the repository baseline. + +> **Grafana note:** Import `docs/ops/scanner-analyzers-grafana-dashboard.json` into your Prometheus-backed Grafana stack to monitor `scanner_analyzer_bench_*` metrics and alert on regressions. + +## Publishing workflow +1. CI (or engineers running locally) execute: + ```bash + dotnet run \ + --project bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj \ + -- \ + --repo-root . \ + --out bench/Scanner.Analyzers/baseline.csv \ + --json out/bench/scanner-analyzers/latest.json \ + --prom out/bench/scanner-analyzers/latest.prom \ + --commit "$(git rev-parse HEAD)" \ + --environment "${CI_ENVIRONMENT_NAME:-local}" + ``` +2. Publish the artefacts (`baseline.csv`, `latest.json`, `latest.prom`) to `bench-artifacts//`. +3. Promtail (or the CI job) pushes `latest.prom` into Prometheus; JSON lands in long-term storage for workbook snapshots. +4. The harness exits non-zero if: + - `max_ms` for any scenario breaches its configured threshold; or + - `max_ms` regresses ≥ 20 % versus `baseline.csv`. + +## Grafana dashboard +- Import `docs/ops/scanner-analyzers-grafana-dashboard.json`. +- Point the template variable `datasource` to the Prometheus instance ingesting `scanner_analyzer_bench_*` metrics. +- Panels: + - **Max Duration (ms)** – compares live runs vs baseline. + - **Regression Ratio vs Limit** – plots `(max / baseline_max - 1) * 100`. + - **Breached Scenarios** – stat panel sourced from `scanner_analyzer_bench_regression_breached`. + +## Alerting & on-call response +- **Primary alert**: fire when `scanner_analyzer_bench_regression_ratio{scenario=~".+"} >= 1.20` for 2 consecutive samples (10 min default). Suggested PromQL: + ``` + max_over_time(scanner_analyzer_bench_regression_ratio[10m]) >= 1.20 + ``` +- Suppress duplicates using the `scenario` label. +- Pager payload should include `scenario`, `max_ms`, `baseline_max_ms`, and `commit`. +- Immediate triage steps: + 1. Check `latest.json` artefact for the failing scenario – confirm commit and environment. + 2. Re-run the harness with `--captured-at` and `--baseline` pointing at the last known good CSV to verify determinism. + 3. If regression persists, open an incident ticket tagged `scanner-analyzer-perf` and page the owning language guild. + 4. Roll back the offending change or update the baseline after sign-off from the guild lead and Perf captain. + +Document the outcome in `docs/12_PERFORMANCE_WORKBOOK.md` (section 8) so trendlines reflect any accepted regressions. diff --git a/docs/ops/scanner-rustfs-migration.md b/docs/ops/scanner-rustfs-migration.md new file mode 100644 index 00000000..28d8081d --- /dev/null +++ b/docs/ops/scanner-rustfs-migration.md @@ -0,0 +1,88 @@ +# Scanner Artifact Store Migration (MinIO → RustFS) + +## Overview + +Sprint 11 introduces **RustFS** as the default artifact store for the Scanner plane. Existing +deployments running MinIO (or any S3-compatible backend) must migrate stored SBOM artefacts to RustFS +before switching the Scanner hosts to `scanner.artifactStore.driver = "rustfs"`. + +This runbook covers the recommended migration workflow and validation steps. + +## Prerequisites + +- RustFS service deployed and reachable from the Scanner control plane (`http(s)://rustfs:8080`). +- Existing MinIO/S3 credentials with read access to the current bucket. +- CLI environment with the StellaOps source tree (for the migration tool) and `dotnet 10` SDK. +- Maintenance window sized to copy all artefacts (migration is read-only on the source bucket). + +## 1. Snapshot source bucket (optional but recommended) + +If the MinIO deployment offers versioning or snapshots, take one before migrating. For non-versioned +deployments, capture an external backup (e.g., `mc mirror` to offline storage). + +## 2. Dry-run the migrator + +``` +dotnet run --project tools/RustFsMigrator -- \ + --s3-bucket scanner-artifacts \ + --s3-endpoint http://stellaops-minio:9000 \ + --s3-access-key stellaops \ + --s3-secret-key dev-minio-secret \ + --rustfs-endpoint http://stellaops-rustfs:8080 \ + --rustfs-bucket scanner-artifacts \ + --prefix scanner/ \ + --dry-run +``` + +The dry-run enumerates keys and reports the object count without writing to RustFS. Use this to +estimate migration time. + +## 3. Execute migration + +Remove the `--dry-run` flag to copy data. Optional flags: + +- `--immutable` – mark all migrated objects as immutable (`X-RustFS-Immutable`). +- `--retain-days 365` – request retention (in days) via `X-RustFS-Retain-Seconds`. +- `--rustfs-api-key-header` / `--rustfs-api-key` – provide auth headers when RustFS is protected. + +The tool streams each object from S3 and performs an idempotent `PUT` to RustFS preserving the key +structure (e.g., `scanner/layers//sbom.cdx.json.zst`). + +## 4. Verify sample objects + +Pick a handful of SBOM digests and confirm: + +1. `GET /api/v1/buckets//objects/` returns the expected payload (size + SHA-256). +2. Scanner WebService configured with `scanner.artifactStore.driver = "rustfs"` can fetch the same + artefacts (Smoke test: `GET /api/v1/scanner/sboms/?format=cdx-json`). + +## 5. Switch Scanner hosts + +Update configuration (Helm/Compose/environment) to set: + +``` +scanner: + artifactStore: + driver: rustfs + endpoint: http://stellaops-rustfs:8080 + bucket: scanner-artifacts + timeoutSeconds: 30 +``` + +Redeploy Scanner WebService and Worker. Monitor logs for `RustFS` upload/download messages and +Prometheus scrape (`rustfs_requests_total`). + +## 6. Cleanup legacy MinIO (optional) + +After a complete migration and validation period, decommission the MinIO bucket or repurpose it for +other components (Concelier still supports S3). Ensure backups reference RustFS snapshots going +forward. + +## Troubleshooting + +- **Uploads fail (HTTP 4xx/5xx):** Check RustFS logs and confirm API key headers. Re-run the migrator + for the affected keys. +- **Missing objects post-cutover:** Re-run the migrator with the specific `--prefix`. The tool is + idempotent and safely overwrites existing objects. +- **Performance tuning:** Run multiple instances of the migrator with disjoint prefixes if needed; the + RustFS API is stateless and supports parallel PUTs. diff --git a/docs/schemas/policy-preview-sample@1.json b/docs/schemas/policy-preview-sample@1.json new file mode 100644 index 00000000..5caa14a1 --- /dev/null +++ b/docs/schemas/policy-preview-sample@1.json @@ -0,0 +1,314 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.stella-ops.org/policy/policy-preview-sample@1.json", + "title": "Policy Preview Sample", + "type": "object", + "additionalProperties": false, + "required": [ + "previewRequest", + "previewResponse" + ], + "properties": { + "previewRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "imageDigest", + "findings" + ], + "properties": { + "imageDigest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "findings": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/finding" + } + }, + "baseline": { + "type": "array", + "items": { + "$ref": "#/$defs/baselineVerdict" + } + } + } + }, + "previewResponse": { + "type": "object", + "additionalProperties": false, + "required": [ + "success", + "policyDigest", + "revisionId", + "changed", + "diffs", + "issues" + ], + "properties": { + "success": { + "type": "boolean" + }, + "policyDigest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "revisionId": { + "type": "string" + }, + "changed": { + "type": "integer", + "minimum": 0 + }, + "diffs": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "findingId", + "baseline", + "projected", + "changed" + ], + "properties": { + "findingId": { + "type": "string" + }, + "baseline": { + "$ref": "#/$defs/baselineVerdict" + }, + "projected": { + "$ref": "#/$defs/projectedVerdict" + }, + "changed": { + "type": "boolean" + } + } + } + }, + "issues": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "message", + "severity", + "path" + ], + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "path": { + "type": "string" + } + } + } + } + } + } + }, + "$defs": { + "finding": { + "type": "object", + "required": [ + "id", + "severity", + "source" + ], + "properties": { + "id": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "source": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + }, + "inputs": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "type": "string", + "maxLength": 64 + }, + "additionalProperties": { + "type": "number" + } + }, + "baselineVerdict": { + "type": "object", + "additionalProperties": false, + "required": [ + "findingId", + "status", + "configVersion", + "score" + ], + "properties": { + "findingId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "Pass", + "Blocked", + "Warned", + "Ignored", + "Deferred", + "Escalated", + "RequiresVex" + ] + }, + "ruleName": { + "type": [ + "string", + "null" + ] + }, + "ruleAction": { + "type": [ + "string", + "null" + ] + }, + "notes": { + "type": [ + "string", + "null" + ] + }, + "score": { + "type": "number" + }, + "configVersion": { + "type": "string" + }, + "inputs": { + "$ref": "#/$defs/inputs" + }, + "quietedBy": { + "type": [ + "string", + "null" + ] + }, + "quiet": { + "type": "boolean" + }, + "unknownConfidence": { + "type": "number", + "minimum": 0 + }, + "confidenceBand": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "unspecified" + ] + }, + "unknownAgeDays": { + "type": "number", + "minimum": 0 + }, + "sourceTrust": { + "type": "string" + }, + "reachability": { + "type": "string", + "enum": [ + "unknown", + "runtime", + "entrypoint", + "direct", + "indirect", + "unreachable" + ] + } + } + }, + "projectedVerdict": { + "allOf": [ + { + "$ref": "#/$defs/baselineVerdict" + }, + { + "type": "object", + "required": [ + "ruleName", + "ruleAction", + "unknownConfidence", + "confidenceBand", + "unknownAgeDays", + "sourceTrust", + "reachability" + ], + "properties": { + "ruleName": { + "type": "string" + }, + "ruleAction": { + "type": "string" + }, + "unknownConfidence": { + "type": "number", + "minimum": 0 + }, + "confidenceBand": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "unspecified" + ] + }, + "unknownAgeDays": { + "type": "number", + "minimum": 0 + }, + "sourceTrust": { + "type": "string" + }, + "reachability": { + "type": "string", + "enum": [ + "unknown", + "runtime", + "entrypoint", + "direct", + "indirect", + "unreachable" + ] + } + } + } + ] + } + } +} diff --git a/docs/schemas/policy-report-sample@1.json b/docs/schemas/policy-report-sample@1.json new file mode 100644 index 00000000..5f322111 --- /dev/null +++ b/docs/schemas/policy-report-sample@1.json @@ -0,0 +1,396 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.stella-ops.org/policy/policy-report-sample@1.json", + "title": "Policy Report Sample", + "type": "object", + "additionalProperties": false, + "required": [ + "reportRequest", + "reportResponse" + ], + "properties": { + "reportRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "imageDigest", + "findings" + ], + "properties": { + "imageDigest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "findings": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/finding" + } + }, + "baseline": { + "type": "array", + "items": { + "$ref": "#/$defs/baselineVerdict" + } + } + } + }, + "reportResponse": { + "type": "object", + "additionalProperties": false, + "required": [ + "report", + "dsse" + ], + "properties": { + "report": { + "type": "object", + "additionalProperties": false, + "required": [ + "reportId", + "imageDigest", + "generatedAt", + "verdict", + "policy", + "summary", + "verdicts", + "issues" + ], + "properties": { + "reportId": { + "type": "string" + }, + "imageDigest": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "generatedAt": { + "type": "string", + "format": "date-time" + }, + "verdict": { + "type": "string" + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "revisionId", + "digest" + ], + "properties": { + "revisionId": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "total", + "blocked", + "warned", + "ignored", + "quieted" + ], + "properties": { + "total": { + "type": "integer", + "minimum": 0 + }, + "blocked": { + "type": "integer", + "minimum": 0 + }, + "warned": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "quieted": { + "type": "integer", + "minimum": 0 + } + } + }, + "verdicts": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/projectedVerdict" + } + }, + "issues": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "code", + "message", + "severity", + "path" + ], + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "path": { + "type": "string" + } + } + } + } + } + }, + "dsse": { + "type": "object", + "additionalProperties": false, + "required": [ + "payloadType", + "payload", + "signatures" + ], + "properties": { + "payloadType": { + "type": "string" + }, + "payload": { + "type": "string" + }, + "signatures": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "keyId", + "algorithm", + "signature" + ], + "properties": { + "keyId": { + "type": "string" + }, + "algorithm": { + "type": "string" + }, + "signature": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "$defs": { + "finding": { + "type": "object", + "required": [ + "id", + "severity", + "source" + ], + "properties": { + "id": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "source": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + }, + "inputs": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "type": "string", + "maxLength": 64 + }, + "additionalProperties": { + "type": "number" + } + }, + "baselineVerdict": { + "type": "object", + "additionalProperties": false, + "required": [ + "findingId", + "status", + "configVersion", + "score" + ], + "properties": { + "findingId": { + "type": "string" + }, + "status": { + "type": "string", + "enum": [ + "Pass", + "Blocked", + "Warned", + "Ignored", + "Deferred", + "Escalated", + "RequiresVex" + ] + }, + "ruleName": { + "type": [ + "string", + "null" + ] + }, + "ruleAction": { + "type": [ + "string", + "null" + ] + }, + "notes": { + "type": [ + "string", + "null" + ] + }, + "score": { + "type": "number" + }, + "configVersion": { + "type": "string" + }, + "inputs": { + "$ref": "#/$defs/inputs" + }, + "quietedBy": { + "type": [ + "string", + "null" + ] + }, + "quiet": { + "type": "boolean" + }, + "unknownConfidence": { + "type": "number", + "minimum": 0 + }, + "confidenceBand": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "unspecified" + ] + }, + "unknownAgeDays": { + "type": "number", + "minimum": 0 + }, + "sourceTrust": { + "type": "string" + }, + "reachability": { + "type": "string", + "enum": [ + "unknown", + "runtime", + "entrypoint", + "direct", + "indirect", + "unreachable" + ] + } + } + }, + "projectedVerdict": { + "allOf": [ + { + "$ref": "#/$defs/baselineVerdict" + }, + { + "type": "object", + "required": [ + "ruleName", + "ruleAction", + "unknownConfidence", + "confidenceBand", + "unknownAgeDays", + "sourceTrust", + "reachability" + ], + "properties": { + "ruleName": { + "type": "string" + }, + "ruleAction": { + "type": "string" + }, + "unknownConfidence": { + "type": "number", + "minimum": 0 + }, + "confidenceBand": { + "type": "string", + "enum": [ + "low", + "medium", + "high", + "unspecified" + ] + }, + "unknownAgeDays": { + "type": "number", + "minimum": 0 + }, + "sourceTrust": { + "type": "string" + }, + "reachability": { + "type": "string", + "enum": [ + "unknown", + "runtime", + "entrypoint", + "direct", + "indirect", + "unreachable" + ] + } + } + } + ] + } + } +} diff --git a/etc/authority.yaml.sample b/etc/authority.yaml.sample index 27878656..3f3aec62 100644 --- a/etc/authority.yaml.sample +++ b/etc/authority.yaml.sample @@ -151,7 +151,7 @@ security: requireChainValidation: true rotationGrace: "00:15:00" enforceForAudiences: - - "signer" + - "signer" # Requests for these audiences force mTLS sender constraints allowedSanTypes: - "dns" - "uri" diff --git a/ops/devops/TASKS.md b/ops/devops/TASKS.md index 40a48c16..5a7a072e 100644 --- a/ops/devops/TASKS.md +++ b/ops/devops/TASKS.md @@ -6,8 +6,9 @@ | DEVOPS-SCANNER-09-204 | DONE (2025-10-21) | DevOps Guild, Scanner WebService Guild | SCANNER-EVENTS-15-201 | Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. | Compose/Helm configs ship enabled Redis event publishing with documented overrides; lint jobs updated; docs cross-link to new knobs. | | DEVOPS-SCANNER-09-205 | DONE (2025-10-21) | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-204 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | CI job reads Redis stream during scanner smoke deploy, confirms Notify ingestion via API, alerts on failure. | | DEVOPS-PERF-10-001 | DONE | DevOps Guild | BENCH-SCANNER-10-001 | Add perf smoke job (SBOM compose <5 s target) to CI. | CI job runs sample build verifying <5 s; alerts configured. | -| DEVOPS-PERF-10-002 | TODO | DevOps Guild | BENCH-SCANNER-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | CI exports JSON for dashboards; Grafana panel wired; Ops on-call doc updated with alert hook. | -| DEVOPS-REL-14-001 | TODO | DevOps Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. | CI pipeline produces signed images + SBOM/attestations, manifests published with verified hashes, docs updated. | +| DEVOPS-PERF-10-002 | DONE (2025-10-23) | DevOps Guild | BENCH-SCANNER-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | CI exports JSON for dashboards; Grafana panel wired; Ops on-call doc updated with alert hook. | +| DEVOPS-REL-14-001 | TODO | DevOps Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. | CI pipeline produces signed images + SBOM/attestations, manifests published with verified hashes, docs updated. | +| DEVOPS-REL-14-004 | TODO | DevOps Guild, Scanner Guild | DEVOPS-REL-14-001, SCANNER-ANALYZERS-LANG-10-309P | Extend release/offline smoke jobs to exercise the Python analyzer plug-in (warm/cold scans, determinism, signature checks). | Release/Offline pipelines run Python analyzer smoke suite; alerts hooked; docs updated with new coverage matrix. | | DEVOPS-REL-17-002 | TODO | DevOps Guild | DEVOPS-REL-14-001, SCANNER-EMIT-17-701 | Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. | CI job writes `.debug` files under `artifacts/debug/.build-id/`, manifest + checksums published, offline kit includes cache, smoke job proves symbol lookup via build-id. | | DEVOPS-MIRROR-08-001 | DONE (2025-10-19) | DevOps Guild | DEVOPS-REL-14-001 | Stand up managed mirror profiles for `*.stella-ops.org` (Concelier/Excititor), including Helm/Compose overlays, multi-tenant secrets, CDN caching, and sync documentation. | Infra overlays committed, CI smoke deploy hits mirror endpoints, runbooks published for downstream sync and quota management. | | DEVOPS-SEC-10-301 | DONE (2025-10-20) | DevOps Guild | Wave 0A complete | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs. | Dependencies bumped to patched releases, audit logs free of NU1902/NU1903 warnings, regression tests green, change log documents upgrade guidance. | diff --git a/ops/offline-kit/TASKS.md b/ops/offline-kit/TASKS.md index 19a19e68..ee35af97 100644 --- a/ops/offline-kit/TASKS.md +++ b/ops/offline-kit/TASKS.md @@ -5,3 +5,4 @@ | DEVOPS-OFFLINE-14-002 | TODO | Offline Kit Guild | DEVOPS-REL-14-001 | Build offline kit packaging workflow (artifact bundling, manifest generation, signature verification). | Offline tarball generated with manifest + checksums + signatures; import script verifies integrity; docs updated. | | DEVOPS-OFFLINE-18-003 | TODO | Offline Kit Guild, UX Specialist | DEVOPS-OFFLINE-14-002 | Capture Angular workspace npm cache + Chromium bundle in Offline Kit (`out/offline-kit/web/`) and document refresh cadence. | Web cache directory added to kit manifest; documentation updated with `npm run ci:install`/`verify:chromium` workflow; periodic refresh SOP recorded in Offline Kit guide. | | DEVOPS-OFFLINE-18-004 | DONE (2025-10-22) | Offline Kit Guild, Scanner Guild | DEVOPS-OFFLINE-18-003, SCANNER-ANALYZERS-LANG-10-309G | Rebuild Offline Kit bundle with Go analyzer plug-in and updated manifest/signature set. | Kit tarball includes Go analyzer artifacts; manifest/signature refreshed; verification steps executed and logged; docs updated with new bundle version. | +| DEVOPS-OFFLINE-18-005 | TODO | Offline Kit Guild, Scanner Guild | DEVOPS-REL-14-004, SCANNER-ANALYZERS-LANG-10-309P | Repackage Offline Kit with Python analyzer plug-in artefacts and refreshed manifest/signature set. | Kit tarball includes Python analyzer DLL/PDB/manifest; signature + manifest updated; Offline Kit guide references Python coverage; smoke import validated. | diff --git a/samples/TASKS.md b/samples/TASKS.md index d4d98279..eebddcb3 100644 --- a/samples/TASKS.md +++ b/samples/TASKS.md @@ -3,4 +3,4 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SAMPLES-10-001 | DONE | Samples Guild, Scanner Team | SCANNER-EMIT-10-605 | Curate sample images (nginx, alpine+busybox, distroless+go, .NET AOT, python venv, npm monorepo) with expected SBOM/BOM-Index sidecars. | Samples committed under `samples/`; golden SBOM/BOM-Index files present; documented usage. | -| SAMPLES-13-004 | TODO | Samples Guild, Policy Guild | POLICY-CORE-09-006, UI-POLICY-13-007 | Add policy preview/report fixtures showing confidence bands and unknown-age tags. | Confidence sample (`samples/policy/policy-preview-unknown.json`) reviewed, documented usage in UI dev guide, ajv validation hook updated. | +| SAMPLES-13-004 | DONE (2025-10-23) | Samples Guild, Policy Guild | POLICY-CORE-09-006, UI-POLICY-13-007 | Add policy preview/report fixtures showing confidence bands and unknown-age tags. | Confidence sample (`samples/policy/policy-preview-unknown.json`) reviewed, documented usage in UI dev guide, ajv validation hook updated. | diff --git a/samples/policy/policy-preview-unknown.json b/samples/policy/policy-preview-unknown.json index b9e3ae3e..ae26acd7 100644 --- a/samples/policy/policy-preview-unknown.json +++ b/samples/policy/policy-preview-unknown.json @@ -1,98 +1,151 @@ -{ - "previewRequest": { - "imageDigest": "sha256:7dbe0c9a5d4f1c8184007e9d94dbe55928f8a2db5ab9c1c2d4a2f7bbcdfe1234", - "findings": [ - { - "id": "library:pkg/openssl@1.1.1w", - "severity": "Unknown", - "source": "NVD", - "tags": [ - "trust:vendor", - "reachability:unknown", - "unknown-age-days:5" - ] - }, - { - "id": "library:pkg/zlib@1.3.1", - "severity": "High", - "source": "NVD", - "tags": [ - "state:unknown", - "reachability:runtime", - "unknown-since:2025-10-10T00:00:00Z", - "observed-at:2025-10-19T12:00:00Z" - ] - } - ] - }, - "previewResponse": { - "success": true, - "policyDigest": "8a0f72f8dc5c51c46991db3bba34e9b3c0c8e944a7a6d0a9c29a9aa6b8439876", - "revisionId": "rev-42", - "changed": 2, - "diffs": [ - { - "findingId": "library:pkg/openssl@1.1.1w", - "baseline": { - "findingId": "library:pkg/openssl@1.1.1w", - "status": "Pass", - "score": 0, - "configVersion": "1.0" - }, - "projected": { - "findingId": "library:pkg/openssl@1.1.1w", - "status": "Blocked", - "ruleName": "Block vendor unknowns", - "ruleAction": "block", - "score": 19.5, - "configVersion": "1.0", - "inputs": { - "severityWeight": 50, - "trustWeight": 0.65, - "reachabilityWeight": 0.6, - "baseScore": 19.5, - "trustWeight.vendor": 0.65, - "reachability.unknown": 0.6, - "unknownConfidence": 0.55, - "unknownAgeDays": 5 - }, - "unknownConfidence": 0.55, - "confidenceBand": "medium", - "unknownAgeDays": 5 - }, - "changed": true - }, - { - "findingId": "library:pkg/zlib@1.3.1", - "baseline": { - "findingId": "library:pkg/zlib@1.3.1", - "status": "Pass", - "score": 0, - "configVersion": "1.0" - }, - "projected": { - "findingId": "library:pkg/zlib@1.3.1", - "status": "Warned", - "ruleName": "Runtime mitigation required", - "ruleAction": "warn", - "score": 33.75, - "configVersion": "1.0", - "inputs": { - "severityWeight": 75, - "trustWeight": 1, - "reachabilityWeight": 0.45, - "baseScore": 33.75, - "reachability.runtime": 0.45, - "warnPenalty": 15, - "unknownConfidence": 0.35, - "unknownAgeDays": 9 - }, - "unknownConfidence": 0.35, - "confidenceBand": "medium", - "unknownAgeDays": 9 - }, - "changed": true - } - ] - } -} +{ + "previewRequest": { + "imageDigest": "sha256:7dbe0c9a5d4f1c8184007e9d94dbe55928f8a2db5ab9c1c2d4a2f7bbcdfe1234", + "findings": [ + { + "id": "library:pkg/openssl@1.1.1w", + "severity": "Unknown", + "source": "NVD", + "tags": [ + "trust:vendor", + "reachability:unknown", + "unknown-age-days:5" + ] + }, + { + "id": "library:pkg/zlib@1.3.1", + "severity": "High", + "source": "NVD", + "tags": [ + "state:unknown", + "reachability:runtime", + "unknown-since:2025-10-10T00:00:00Z", + "observed-at:2025-10-19T12:00:00Z" + ] + } + ], + "baseline": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Pass", + "score": 0, + "configVersion": "1.0", + "inputs": { + "severityWeight": 25, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 11.25 + }, + "quiet": false + }, + { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Pass", + "score": 0, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75 + }, + "quiet": false + } + ] + }, + "previewResponse": { + "success": true, + "policyDigest": "8a0f72f8dc5c51c46991db3bba34e9b3c0c8e944a7a6d0a9c29a9aa6b8439876", + "revisionId": "rev-42", + "changed": 2, + "diffs": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "baseline": { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Pass", + "score": 0, + "configVersion": "1.0", + "inputs": { + "severityWeight": 25, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 11.25 + }, + "quiet": false + }, + "projected": { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Blocked", + "ruleName": "Block vendor unknowns", + "ruleAction": "block", + "notes": "Unknown vendor telemetry — medium confidence band.", + "score": 19.5, + "configVersion": "1.0", + "inputs": { + "severityWeight": 50, + "trustWeight": 0.65, + "reachabilityWeight": 0.6, + "baseScore": 19.5, + "trustWeight.vendor": 0.65, + "reachability.unknown": 0.6, + "unknownConfidence": 0.55, + "unknownAgeDays": 5 + }, + "quietedBy": null, + "quiet": false, + "unknownConfidence": 0.55, + "confidenceBand": "medium", + "unknownAgeDays": 5, + "sourceTrust": "vendor", + "reachability": "unknown" + }, + "changed": true + }, + { + "findingId": "library:pkg/zlib@1.3.1", + "baseline": { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Pass", + "score": 0, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75 + }, + "quiet": false + }, + "projected": { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Warned", + "ruleName": "Runtime mitigation required", + "ruleAction": "warn", + "notes": "Runtime reachable unknown — mitigation window required.", + "score": 18.75, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75, + "reachability.runtime": 0.45, + "warnPenalty": 15, + "unknownConfidence": 0.35, + "unknownAgeDays": 13 + }, + "quietedBy": null, + "quiet": false, + "unknownConfidence": 0.35, + "confidenceBand": "medium", + "unknownAgeDays": 13, + "sourceTrust": "NVD", + "reachability": "runtime" + }, + "changed": true + } + ], + "issues": [] + } +} diff --git a/samples/policy/policy-report-unknown.json b/samples/policy/policy-report-unknown.json new file mode 100644 index 00000000..8843c294 --- /dev/null +++ b/samples/policy/policy-report-unknown.json @@ -0,0 +1,141 @@ +{ + "reportRequest": { + "imageDigest": "sha256:7dbe0c9a5d4f1c8184007e9d94dbe55928f8a2db5ab9c1c2d4a2f7bbcdfe1234", + "findings": [ + { + "id": "library:pkg/openssl@1.1.1w", + "severity": "Unknown", + "source": "NVD", + "tags": [ + "trust:vendor", + "reachability:unknown", + "unknown-age-days:5" + ] + }, + { + "id": "library:pkg/zlib@1.3.1", + "severity": "High", + "source": "NVD", + "tags": [ + "state:unknown", + "reachability:runtime", + "unknown-since:2025-10-10T00:00:00Z", + "observed-at:2025-10-19T12:00:00Z" + ] + } + ], + "baseline": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Pass", + "score": 0, + "configVersion": "1.0", + "inputs": { + "severityWeight": 25, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 11.25 + }, + "quiet": false + }, + { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Pass", + "score": 0, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75 + }, + "quiet": false + } + ] + }, + "reportResponse": { + "report": { + "reportId": "report-9f8cde21aab54321", + "imageDigest": "sha256:7dbe0c9a5d4f1c8184007e9d94dbe55928f8a2db5ab9c1c2d4a2f7bbcdfe1234", + "generatedAt": "2025-10-23T15:32:22Z", + "verdict": "blocked", + "policy": { + "revisionId": "rev-42", + "digest": "8a0f72f8dc5c51c46991db3bba34e9b3c0c8e944a7a6d0a9c29a9aa6b8439876" + }, + "summary": { + "total": 2, + "blocked": 1, + "warned": 1, + "ignored": 0, + "quieted": 0 + }, + "verdicts": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Blocked", + "ruleName": "Block vendor unknowns", + "ruleAction": "block", + "notes": "Unknown vendor telemetry — medium confidence band.", + "score": 19.5, + "configVersion": "1.0", + "inputs": { + "severityWeight": 50, + "trustWeight": 0.65, + "reachabilityWeight": 0.6, + "baseScore": 19.5, + "trustWeight.vendor": 0.65, + "reachability.unknown": 0.6, + "unknownConfidence": 0.55, + "unknownAgeDays": 5 + }, + "quietedBy": null, + "quiet": false, + "unknownConfidence": 0.55, + "confidenceBand": "medium", + "unknownAgeDays": 5, + "sourceTrust": "vendor", + "reachability": "unknown" + }, + { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Warned", + "ruleName": "Runtime mitigation required", + "ruleAction": "warn", + "notes": "Runtime reachable unknown — mitigation window required.", + "score": 18.75, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75, + "reachability.runtime": 0.45, + "warnPenalty": 15, + "unknownConfidence": 0.35, + "unknownAgeDays": 13 + }, + "quietedBy": null, + "quiet": false, + "unknownConfidence": 0.35, + "confidenceBand": "medium", + "unknownAgeDays": 13, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ], + "issues": [] + }, + "dsse": { + "payloadType": "application/vnd.stellaops.report+json", + "payload": "eyJyZXBvcnQiOnsicmVwb3J0SWQiOiJyZXBvcnQtOWY4Y2RlMjFhYWI1NDMyMSJ9fQ==", + "signatures": [ + { + "keyId": "scanner-report-signing", + "algorithm": "hs256", + "signature": "MEQCIGHscnJ2bm9wYXlsb2FkZXIAIjANBgkqhkiG9w0BAQsFAAOCAQEASmFja3Nvbk1ldGE=" + } + ] + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs index d36e0f69..e3ae26f1 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs @@ -50,6 +50,7 @@ public class ClientCredentialsHandlersTests allowedScopes: "jobs:read"); var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); var handler = new ValidateClientCredentialsHandler( new TestClientStore(clientDocument), registry, @@ -59,6 +60,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, new NoopCertificateValidator(), new HttpContextAccessor(), + options, NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:write"); @@ -80,6 +82,7 @@ public class ClientCredentialsHandlersTests allowedScopes: "jobs:read jobs:trigger"); var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var options = TestHelpers.CreateAuthorityOptions(); var handler = new ValidateClientCredentialsHandler( new TestClientStore(clientDocument), registry, @@ -89,6 +92,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, new NoopCertificateValidator(), new HttpContextAccessor(), + options, NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); @@ -114,6 +118,7 @@ public class ClientCredentialsHandlersTests var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); var sink = new TestAuthEventSink(); + var options = TestHelpers.CreateAuthorityOptions(); var handler = new ValidateClientCredentialsHandler( new TestClientStore(clientDocument), registry, @@ -123,6 +128,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, new NoopCertificateValidator(), new HttpContextAccessor(), + options, NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); @@ -139,12 +145,11 @@ public class ClientCredentialsHandlersTests [Fact] public async Task ValidateDpopProof_AllowsSenderConstrainedClient() { - var options = new StellaOpsAuthorityOptions + var options = TestHelpers.CreateAuthorityOptions(opts => { - Issuer = new Uri("https://authority.test") - }; - options.Security.SenderConstraints.Dpop.Enabled = true; - options.Security.SenderConstraints.Dpop.Nonce.Enabled = false; + opts.Security.SenderConstraints.Dpop.Enabled = true; + opts.Security.SenderConstraints.Dpop.Nonce.Enabled = false; + }); var clientDocument = CreateClient( secret: "s3cr3t!", @@ -214,6 +219,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, new NoopCertificateValidator(), new HttpContextAccessor(), + options, NullLogger.Instance); await validateHandler.HandleAsync(validateContext); @@ -389,6 +395,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, validator, httpContextAccessor, + options, NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); @@ -435,6 +442,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, validator, httpContextAccessor, + options, NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); @@ -446,6 +454,94 @@ public class ClientCredentialsHandlersTests Assert.Equal(OpenIddictConstants.Errors.InvalidClient, context.Error); } + [Fact] + public async Task ValidateClientCredentials_Rejects_WhenAudienceRequiresMtlsButClientConfiguredForDpop() + { + var options = TestHelpers.CreateAuthorityOptions(opts => + { + opts.Security.SenderConstraints.Mtls.Enabled = true; + opts.Security.SenderConstraints.Mtls.EnforceForAudiences.Clear(); + opts.Security.SenderConstraints.Mtls.EnforceForAudiences.Add("signer"); + }); + + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read", + allowedAudiences: "signer"); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Dpop; + clientDocument.Properties[AuthorityClientMetadataKeys.SenderConstraint] = AuthoritySenderConstraintKinds.Dpop; + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidClient, context.Error); + Assert.Equal("Requested audiences require mutual TLS sender constraint.", context.ErrorDescription); + } + + [Fact] + public async Task ValidateClientCredentials_RequiresMtlsWhenAudienceMatchesEnforcement() + { + var options = TestHelpers.CreateAuthorityOptions(opts => + { + opts.Security.SenderConstraints.Mtls.Enabled = true; + opts.Security.SenderConstraints.Mtls.EnforceForAudiences.Clear(); + opts.Security.SenderConstraints.Mtls.EnforceForAudiences.Add("signer"); + }); + + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read", + allowedAudiences: "signer"); + clientDocument.CertificateBindings.Add(new AuthorityClientCertificateBinding + { + Thumbprint = "DEADBEEF" + }); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var certificateValidator = new RecordingCertificateValidator(); + var httpContextAccessor = new HttpContextAccessor { HttpContext = new DefaultHttpContext() }; + + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + certificateValidator, + httpContextAccessor, + options, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidClient, context.Error); + Assert.Equal("client_certificate_required", context.ErrorDescription); + Assert.True(certificateValidator.Invoked); + } + [Fact] public async Task HandleClientCredentials_PersistsTokenAndEnrichesClaims() { @@ -462,6 +558,7 @@ public class ClientCredentialsHandlersTests var sessionAccessor = new NullMongoSessionAccessor(); var authSink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); + var options = TestHelpers.CreateAuthorityOptions(); var validateHandler = new ValidateClientCredentialsHandler( new TestClientStore(clientDocument), registry, @@ -471,6 +568,7 @@ public class ClientCredentialsHandlersTests TimeProvider.System, new NoopCertificateValidator(), new HttpContextAccessor(), + options, NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger"); @@ -828,6 +926,82 @@ public class AuthorityClientCertificateValidatorTests Assert.True(result.Succeeded); Assert.Equal(thumbprint, result.HexThumbprint); } + + [Fact] + public async Task ValidateAsync_Rejects_WhenBindingSubjectMismatch() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Mtls.Enabled = true; + options.Security.SenderConstraints.Mtls.RequireChainValidation = false; + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=mtls-client", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName("client.mtls.test"); + request.CertificateExtensions.Add(sanBuilder.Build()); + using var certificate = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddMinutes(5)); + + var clientDocument = CreateClient(); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Mtls; + clientDocument.CertificateBindings.Add(new AuthorityClientCertificateBinding + { + Thumbprint = Convert.ToHexString(certificate.GetCertHash(HashAlgorithmName.SHA256)), + Subject = "CN=different-client" + }); + + var httpContext = new DefaultHttpContext(); + httpContext.Connection.ClientCertificate = certificate; + + var validator = new AuthorityClientCertificateValidator(options, TimeProvider.System, NullLogger.Instance); + var result = await validator.ValidateAsync(httpContext, clientDocument, CancellationToken.None); + + Assert.False(result.Succeeded); + Assert.Equal("certificate_binding_subject_mismatch", result.Error); + } + + [Fact] + public async Task ValidateAsync_Rejects_WhenBindingSansMissing() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Mtls.Enabled = true; + options.Security.SenderConstraints.Mtls.RequireChainValidation = false; + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=mtls-client", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName("client.mtls.test"); + request.CertificateExtensions.Add(sanBuilder.Build()); + using var certificate = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddMinutes(5)); + + var clientDocument = CreateClient(); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Mtls; + clientDocument.CertificateBindings.Add(new AuthorityClientCertificateBinding + { + Thumbprint = Convert.ToHexString(certificate.GetCertHash(HashAlgorithmName.SHA256)), + SubjectAlternativeNames = new List { "spiffe://client" } + }); + + var httpContext = new DefaultHttpContext(); + httpContext.Connection.ClientCertificate = certificate; + + var validator = new AuthorityClientCertificateValidator(options, TimeProvider.System, NullLogger.Instance); + var result = await validator.ValidateAsync(httpContext, clientDocument, CancellationToken.None); + + Assert.False(result.Succeeded); + Assert.Equal("certificate_binding_san_mismatch", result.Error); + } } internal sealed class TestClientStore : IAuthorityClientStore @@ -1011,6 +1185,33 @@ internal sealed class NoopCertificateValidator : IAuthorityClientCertificateVali } } +internal sealed class RecordingCertificateValidator : IAuthorityClientCertificateValidator +{ + public bool Invoked { get; private set; } + + public ValueTask ValidateAsync(HttpContext httpContext, AuthorityClientDocument client, CancellationToken cancellationToken) + { + Invoked = true; + + if (httpContext.Connection.ClientCertificate is null) + { + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("client_certificate_required")); + } + + AuthorityClientCertificateBinding binding; + if (client.CertificateBindings.Count > 0) + { + binding = client.CertificateBindings[0]; + } + else + { + binding = new AuthorityClientCertificateBinding { Thumbprint = "stub" }; + } + + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Success("stub", binding.Thumbprint, binding)); + } +} + internal sealed class NullMongoSessionAccessor : IAuthorityMongoSessionAccessor { public ValueTask GetSessionAsync(CancellationToken cancellationToken = default) @@ -1021,6 +1222,21 @@ internal sealed class NullMongoSessionAccessor : IAuthorityMongoSessionAccessor internal static class TestHelpers { + public static StellaOpsAuthorityOptions CreateAuthorityOptions(Action? configure = null) + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + configure?.Invoke(options); + return options; + } + public static AuthorityClientDocument CreateClient( string? secret = "s3cr3t!", string clientType = "confidential", diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs index 9d07a12a..2141b8d3 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs @@ -64,7 +64,8 @@ public sealed class TokenPersistenceIntegrationTests var metadataAccessor = new TestRateLimiterMetadataAccessor(); await using var scope = provider.CreateAsyncScope(); var sessionAccessor = scope.ServiceProvider.GetRequiredService(); - var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, new NoopCertificateValidator(), new HttpContextAccessor(), NullLogger.Instance); + var options = TestHelpers.CreateAuthorityOptions(); + var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, new NoopCertificateValidator(), new HttpContextAccessor(), options, NullLogger.Instance); var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, sessionAccessor, clock, TestActivitySource, NullLogger.Instance); var persistHandler = new PersistTokensHandler(tokenStore, sessionAccessor, clock, TestActivitySource, NullLogger.Instance); diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs index e5998be6..b8037cdc 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs @@ -10,18 +10,19 @@ using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; using OpenIddict.Abstractions; using OpenIddict.Extensions; -using OpenIddict.Server; -using OpenIddict.Server.AspNetCore; -using MongoDB.Driver; -using StellaOps.Auth.Abstractions; -using StellaOps.Authority.OpenIddict; -using StellaOps.Authority.Plugins.Abstractions; -using StellaOps.Authority.Storage.Mongo.Documents; -using StellaOps.Authority.Storage.Mongo.Sessions; -using StellaOps.Authority.Storage.Mongo.Stores; -using StellaOps.Authority.RateLimiting; -using StellaOps.Authority.Security; -using StellaOps.Cryptography.Audit; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using MongoDB.Driver; +using StellaOps.Auth.Abstractions; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Sessions; +using StellaOps.Authority.Storage.Mongo.Stores; +using StellaOps.Authority.RateLimiting; +using StellaOps.Authority.Security; +using StellaOps.Configuration; +using StellaOps.Cryptography.Audit; namespace StellaOps.Authority.OpenIddict.Handlers; @@ -31,33 +32,36 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle private readonly IAuthorityIdentityProviderRegistry registry; private readonly ActivitySource activitySource; private readonly IAuthEventSink auditSink; - private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; - private readonly TimeProvider timeProvider; - private readonly IAuthorityClientCertificateValidator certificateValidator; - private readonly IHttpContextAccessor httpContextAccessor; - private readonly ILogger logger; + private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; + private readonly TimeProvider timeProvider; + private readonly IAuthorityClientCertificateValidator certificateValidator; + private readonly IHttpContextAccessor httpContextAccessor; + private readonly StellaOpsAuthorityOptions authorityOptions; + private readonly ILogger logger; public ValidateClientCredentialsHandler( IAuthorityClientStore clientStore, IAuthorityIdentityProviderRegistry registry, ActivitySource activitySource, IAuthEventSink auditSink, - IAuthorityRateLimiterMetadataAccessor metadataAccessor, - TimeProvider timeProvider, - IAuthorityClientCertificateValidator certificateValidator, - IHttpContextAccessor httpContextAccessor, - ILogger logger) - { - this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); - this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); - this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); - this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); - this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); - this.certificateValidator = certificateValidator ?? throw new ArgumentNullException(nameof(certificateValidator)); - this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } + IAuthorityRateLimiterMetadataAccessor metadataAccessor, + TimeProvider timeProvider, + IAuthorityClientCertificateValidator certificateValidator, + IHttpContextAccessor httpContextAccessor, + StellaOpsAuthorityOptions authorityOptions, + ILogger logger) + { + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); + this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.certificateValidator = certificateValidator ?? throw new ArgumentNullException(nameof(certificateValidator)); + this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); + this.authorityOptions = authorityOptions ?? throw new ArgumentNullException(nameof(authorityOptions)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } public async ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenRequestContext context) { @@ -124,14 +128,30 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle ? existingConstraint : null; - var normalizedSenderConstraint = !string.IsNullOrWhiteSpace(existingSenderConstraint) - ? existingSenderConstraint - : ClientCredentialHandlerHelpers.NormalizeSenderConstraint(document); - - if (!string.IsNullOrWhiteSpace(normalizedSenderConstraint)) - { - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientSenderConstraintProperty] = normalizedSenderConstraint; - } + var normalizedSenderConstraint = !string.IsNullOrWhiteSpace(existingSenderConstraint) + ? existingSenderConstraint + : ClientCredentialHandlerHelpers.NormalizeSenderConstraint(document); + + var (mtlsRequired, matchedAudiences) = EvaluateMtlsRequirement(context.Request, document); + if (mtlsRequired) + { + if (string.IsNullOrWhiteSpace(normalizedSenderConstraint)) + { + normalizedSenderConstraint = AuthoritySenderConstraintKinds.Mtls; + logger.LogDebug("Enforcing mTLS sender constraint for {ClientId} due to audiences {Audiences}.", document.ClientId, string.Join(",", matchedAudiences)); + } + else if (!string.Equals(normalizedSenderConstraint, AuthoritySenderConstraintKinds.Mtls, StringComparison.Ordinal)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Requested audiences require mutual TLS sender constraint."); + logger.LogWarning("Client credentials validation failed for {ClientId}: mTLS required for audiences {Audiences} but client sender constraint was {Constraint}.", context.ClientId, string.Join(",", matchedAudiences), normalizedSenderConstraint); + return; + } + } + + if (!string.IsNullOrWhiteSpace(normalizedSenderConstraint)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientSenderConstraintProperty] = normalizedSenderConstraint; + } if (string.Equals(normalizedSenderConstraint, AuthoritySenderConstraintKinds.Mtls, StringComparison.Ordinal)) { @@ -281,8 +301,95 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); } - } -} + } + + private (bool Required, string[] Audiences) EvaluateMtlsRequirement(OpenIddictRequest? request, AuthorityClientDocument document) + { + var mtlsOptions = authorityOptions.Security.SenderConstraints.Mtls; + if (!mtlsOptions.Enabled) + { + return (false, Array.Empty()); + } + + var enforcedAudiences = ResolveEnforcedAudiences(mtlsOptions); + if (enforcedAudiences.Count == 0) + { + return (false, Array.Empty()); + } + + static void CollectMatches(IEnumerable values, ISet enforced, HashSet matches) + { + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + var candidate = value.Trim(); + if (candidate.Length == 0) + { + continue; + } + + if (enforced.Contains(candidate)) + { + matches.Add(candidate); + } + } + } + + var matched = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (request?.Resources is { } resources) + { + CollectMatches(resources, enforcedAudiences, matched); + } + + if (request?.Audiences is { } audiences) + { + CollectMatches(audiences, enforcedAudiences, matched); + } + + var configuredAudiences = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.Audiences); + if (configuredAudiences.Count > 0) + { + CollectMatches(configuredAudiences, enforcedAudiences, matched); + } + + return matched.Count == 0 + ? (false, Array.Empty()) + : (true, matched.OrderBy(value => value, StringComparer.OrdinalIgnoreCase).ToArray()); + } + + private static HashSet ResolveEnforcedAudiences(AuthorityMtlsOptions mtlsOptions) + { + if (mtlsOptions.NormalizedAudiences.Count > 0) + { + return new HashSet(mtlsOptions.NormalizedAudiences, StringComparer.OrdinalIgnoreCase); + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var audience in mtlsOptions.EnforceForAudiences) + { + if (string.IsNullOrWhiteSpace(audience)) + { + continue; + } + + var trimmed = audience.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + set.Add(trimmed); + } + + return set; + } +} internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler { diff --git a/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs index bfe0ec17..e491c4bc 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs @@ -1,10 +1,11 @@ using System; -using System.Linq; -using System.Security.Cryptography; -using System.Security.Cryptography.X509Certificates; -using System.Threading; -using System.Threading.Tasks; -using System.Formats.Asn1; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Asn1; using System.Net; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Logging; @@ -145,12 +146,47 @@ internal sealed class AuthorityClientCertificateValidator : IAuthorityClientCert return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_unbound")); } - if (binding.NotBefore is { } bindingNotBefore) - { - var effectiveNotBefore = bindingNotBefore - mtlsOptions.RotationGrace; - if (now < effectiveNotBefore) - { - logger.LogWarning("mTLS validation failed for {ClientId}: certificate binding not active until {NotBefore:o} (grace applied).", client.ClientId, bindingNotBefore); + if (!string.IsNullOrWhiteSpace(binding.Subject) && + !string.Equals(binding.Subject, certificate.Subject, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate subject {Subject} did not match binding subject {BindingSubject}.", client.ClientId, certificate.Subject, binding.Subject); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_subject_mismatch")); + } + + if (!string.IsNullOrWhiteSpace(binding.SerialNumber)) + { + var normalizedCertificateSerial = NormalizeSerialNumber(certificate.SerialNumber); + var normalizedBindingSerial = NormalizeSerialNumber(binding.SerialNumber); + if (!string.Equals(normalizedCertificateSerial, normalizedBindingSerial, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate serial {Serial} did not match binding serial {BindingSerial}.", client.ClientId, normalizedCertificateSerial, normalizedBindingSerial); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_serial_mismatch")); + } + } + + if (!string.IsNullOrWhiteSpace(binding.Issuer) && + !string.Equals(binding.Issuer, certificate.Issuer, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate issuer {Issuer} did not match binding issuer {BindingIssuer}.", client.ClientId, certificate.Issuer, binding.Issuer); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_issuer_mismatch")); + } + + if (binding.SubjectAlternativeNames.Count > 0) + { + var certificateSans = new HashSet(subjectAlternativeNames.Select(san => san.Value), StringComparer.OrdinalIgnoreCase); + if (!binding.SubjectAlternativeNames.All(san => certificateSans.Contains(san))) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate SANs did not include all binding values.", client.ClientId); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_san_mismatch")); + } + } + + if (binding.NotBefore is { } bindingNotBefore) + { + var effectiveNotBefore = bindingNotBefore - mtlsOptions.RotationGrace; + if (now < effectiveNotBefore) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate binding not active until {NotBefore:o} (grace applied).", client.ClientId, bindingNotBefore); return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_inactive")); } } @@ -197,11 +233,11 @@ internal sealed class AuthorityClientCertificateValidator : IAuthorityClientCert } } - private static IReadOnlyList<(string Type, string Value)> GetSubjectAlternativeNames(X509Certificate2 certificate) - { - foreach (var extension in certificate.Extensions) - { - if (!string.Equals(extension.Oid?.Value, "2.5.29.17", StringComparison.Ordinal)) + private static IReadOnlyList<(string Type, string Value)> GetSubjectAlternativeNames(X509Certificate2 certificate) + { + foreach (var extension in certificate.Extensions) + { + if (!string.Equals(extension.Oid?.Value, "2.5.29.17", StringComparison.Ordinal)) { continue; } @@ -256,28 +292,28 @@ internal sealed class AuthorityClientCertificateValidator : IAuthorityClientCert } } - return Array.Empty<(string, string)>(); - } - private bool ValidateCertificateChain(X509Certificate2 certificate) - { - using var chain = new X509Chain - { - ChainPolicy = - { - RevocationMode = X509RevocationMode.NoCheck, - RevocationFlag = X509RevocationFlag.ExcludeRoot, - VerificationFlags = X509VerificationFlags.IgnoreWrongUsage - } - }; - - try - { - return chain.Build(certificate); - } - catch (Exception ex) - { - logger.LogWarning(ex, "mTLS chain validation threw an exception."); - return false; - } - } -} + return Array.Empty<(string, string)>(); + } + + private static string NormalizeSerialNumber(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var buffer = new char[value.Length]; + var length = 0; + foreach (var character in value) + { + if (character is ':' or ' ') + { + continue; + } + + buffer[length++] = char.ToUpperInvariant(character); + } + + return new string(buffer, 0, length); + } +} diff --git a/src/StellaOps.Authority/TASKS.md b/src/StellaOps.Authority/TASKS.md index f2baed13..6877b77f 100644 --- a/src/StellaOps.Authority/TASKS.md +++ b/src/StellaOps.Authority/TASKS.md @@ -23,10 +23,9 @@ | AUTH-PLUGIN-COORD-08-002 | DONE (2025-10-20) | Authority Core, Plugin Platform Guild | PLUGIN-DI-08-001 | Coordinate scoped-service adoption for Authority plug-in registrars and background jobs ahead of PLUGIN-DI-08-002 implementation. | ✅ Workshop completed 2025-10-20 15:00–16:05 UTC with notes/action log in `docs/dev/authority-plugin-di-coordination.md`; ✅ Follow-up backlog updates assigned via documented action items ahead of PLUGIN-DI-08-002 delivery. | | AUTH-DPOP-11-001 | DONE (2025-10-20) | Authority Core & Security Guild | — | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | ✅ Redis-configurable nonce store surfaced via `security.senderConstraints.dpop.nonce` with sample YAML and architecture docs refreshed
✅ High-value audience enforcement uses normalised required audiences to avoid whitespace/case drift
✅ Operator guide updated with Redis-backed nonce snippet and env-var override guidance; integration test already covers nonce challenge | > Remark (2025-10-20): `etc/authority.yaml.sample` gains senderConstraint sections (rate limits, DPoP, mTLS), docs (`docs/ARCHITECTURE_AUTHORITY.md`, `docs/11_AUTHORITY.md`, plan) refreshed. `ResolveNonceAudience` now relies on `NormalizedAudiences` and options trim persisted values. `dotnet test StellaOps.Authority.sln` attempted (2025-10-20 15:12 UTC) but failed on `NU1900` because the mirrored NuGet service index `https://mirrors.ablera.dev/nuget/nuget-mirror/v3/index.json` was unreachable; no project build executed. -| AUTH-MTLS-11-002 | DOING (2025-10-19) | Authority Core & Security Guild | — | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | • Certificate validator scaffold plus cnf stamping present; tokens persist sender thumbprints
• Remaining: provisioning/storage for certificate bindings, SAN/CA validation, introspection propagation, integration tests/docs before marking DONE | -> Remark (2025-10-19): Client provisioning accepts certificate bindings; validator enforces SAN types/CA allow-list with rotation grace; mtls integration tests updated (full suite still blocked by upstream build). +| AUTH-MTLS-11-002 | DONE (2025-10-23) | Authority Core & Security Guild | — | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | ✅ Deterministic provisioning/storage for certificate bindings (thumbprint/subject/issuer/serial/SAN)
✅ Audience enforcement auto-switches to mTLS via `security.senderConstraints.mtls.enforceForAudiences`
✅ Validator matches binding metadata with rotation grace and emits confirmation thumbprints
✅ Introspection returns `cnf.x5t#S256`; docs & sample config refreshed; Authority test suite green | +> Remark (2025-10-23): Audience enforcement now rejects non-mTLS clients targeting high-value audiences; certificate validator checks binding subject/issuer/serial/SAN values and returns deterministic error codes. Docs (`docs/11_AUTHORITY.md`, `docs/ARCHITECTURE_AUTHORITY.md`, `docs/dev/authority-dpop-mtls-plan.md`) and `etc/authority.yaml.sample` updated. `dotnet test src/StellaOps.Authority/StellaOps.Authority.sln` (2025-10-23 18:07 UTC) succeeded. > Remark (2025-10-19, AUTHSTORAGE-MONGO-08-001): Prerequisites re-checked (none outstanding). Session accessor wired through Authority pipeline; stores accept optional sessions; added replica-set election regression test for read-your-write. > Remark (2025-10-19, AUTH-DPOP-11-001): Handler, nonce store, and persistence hooks merged; Redis-backed configuration + end-to-end nonce enforcement still open. (Superseded by 2025-10-20 update above.) -> Remark (2025-10-19, AUTH-MTLS-11-002): Certificate validator + cnf stamping delivered; binding storage, CA/SAN validation, integration suites outstanding before status can move to DONE. > Update status columns (TODO / DOING / DONE / BLOCKED) together with code changes. Always run `dotnet test src/StellaOps.Authority.sln` when touching host logic. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json new file mode 100644 index 00000000..2eea00d4 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json @@ -0,0 +1,110 @@ +[ + { + "analyzerId": "python", + "componentKey": "purl::pkg:pypi/layered@2.0", + "purl": "pkg:pypi/layered@2.0", + "name": "layered", + "version": "2.0", + "type": "pypi", + "usedByEntrypoint": true, + "metadata": { + "author": "Layered Maintainer", + "authorEmail": "maintainer@example.com", + "classifier[0]": "Programming Language :: Python :: 3", + "classifiers": "Programming Language :: Python :: 3", + "distInfoPath": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info", + "editable": "true", + "entryPoints.console_scripts": "layered-cli=layered.cli:main", + "entryPoints.layered.hooks": "register=layered.plugins:register", + "installer": "pip", + "license": "Apache-2.0", + "license.classifier[0]": "License :: OSI Approved :: Apache Software License", + "license.file[0]": "layer2/usr/lib/python3.11/site-packages/LICENSE", + "licenseExpression": "Apache-2.0", + "name": "layered", + "normalizedName": "layered", + "projectUrl": "Documentation, https://example.com/layered/docs", + "provenance": "dist-info", + "record.hashMismatches": "0", + "record.hashedEntries": "8", + "record.ioErrors": "0", + "record.missingFiles": "0", + "record.totalEntries": "9", + "requiresDist": "requests", + "requiresPython": "\u003E=3.9", + "sourceCommit": "abc123", + "sourceSubdirectory": "src/layered", + "sourceUrl": "https://git.example.com/layered", + "sourceVcs": "git", + "summary": "Base layer metadata", + "version": "2.0", + "wheel.generator": "pip 24.0", + "wheel.rootIsPurelib": "true", + "wheel.tags": "py3-none-any", + "wheel.version": "1.0" + }, + "evidence": [ + { + "kind": "file", + "source": "INSTALLER", + "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER" + }, + { + "kind": "file", + "source": "INSTALLER", + "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER" + }, + { + "kind": "file", + "source": "METADATA", + "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA" + }, + { + "kind": "file", + "source": "METADATA", + "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA" + }, + { + "kind": "file", + "source": "RECORD", + "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD" + }, + { + "kind": "file", + "source": "RECORD", + "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD" + }, + { + "kind": "file", + "source": "WHEEL", + "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL" + }, + { + "kind": "file", + "source": "WHEEL", + "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL" + }, + { + "kind": "file", + "source": "entry_points.txt", + "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt" + }, + { + "kind": "file", + "source": "entry_points.txt", + "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt" + }, + { + "kind": "file", + "source": "license", + "locator": "layer2/usr/lib/python3.11/site-packages/LICENSE" + }, + { + "kind": "metadata", + "source": "direct_url.json", + "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/direct_url.json", + "value": "https://git.example.com/layered" + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER new file mode 100644 index 00000000..a1b589e3 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA new file mode 100644 index 00000000..a86f2851 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA @@ -0,0 +1,8 @@ +Metadata-Version: 2.1 +Name: layered +Version: 2.0 +Summary: Base layer metadata +License: Apache-2.0 +Classifier: Programming Language :: Python :: 3 +Requires-Python: >=3.9 +Requires-Dist: requests diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD new file mode 100644 index 00000000..3b0d6854 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD @@ -0,0 +1,9 @@ +layered/__init__.py,sha256=3Q3bv/BWCZLW2p2dVYw8QfAfBYV2YBtuYtT9TIJCmFM=,139 +layered/core.py,sha256=izBXI4cRE/cgf7hgc/gHfTp1OyIIJ23NgUJXaHWVFpU=,80 +layered/cli.py,sha256=xQrTznF7ch6C9qyQALJpsqRTIh9DVCRG4IXoQ1eLLnY=,126 +../../../bin/layered-cli,sha256=6IGTGCEapolFoAUnXGNrWIfPSXU8W7bsZ07DYF/wmNc=,91 +layered-2.0.dist-info/METADATA,sha256=jNEi7xsj4V+SSzOJJToxMoZmZ7gxyto7zuKxCjxUFjk=,193 +layered-2.0.dist-info/WHEEL,sha256=m8MHT7vQnqC5W8H/y4uJdEpx9ijH0jJGpuoaxRbcsQg=,79 +layered-2.0.dist-info/entry_points.txt,sha256=hm8bgJUYe2zoYNATyAsQzQKQTdQtFe4ctbf5kSlxFj0=,47 +layered-2.0.dist-info/INSTALLER,sha256=zuuue4knoyJ+UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg=,4 +layered-2.0.dist-info/RECORD,, diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL new file mode 100644 index 00000000..e0965d4f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: pip 24.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt new file mode 100644 index 00000000..e16cdd74 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +layered-cli=layered.cli:main diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/__init__.py b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/__init__.py new file mode 100644 index 00000000..5bc1bc03 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/__init__.py @@ -0,0 +1,5 @@ +"""Layered package demonstrating merged metadata across layers.""" + +from .core import get_version # noqa: F401 + +__all__ = ["get_version"] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/cli.py b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/cli.py new file mode 100644 index 00000000..6a2c0790 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/cli.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from .core import get_version + + +def main() -> None: + print(f"layered {get_version()}") diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/core.py b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/core.py new file mode 100644 index 00000000..c0ae06a9 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer1/usr/lib/python3.11/site-packages/layered/core.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +def get_version() -> str: + return "2.0" diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/LICENSE b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/LICENSE new file mode 100644 index 00000000..0c14b24b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/LICENSE @@ -0,0 +1 @@ +Apache License Version 2.0 diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER new file mode 100644 index 00000000..a1b589e3 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA new file mode 100644 index 00000000..6142e746 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA @@ -0,0 +1,10 @@ +Metadata-Version: 2.1 +Name: layered +Version: 2.0 +Summary: Overlay metadata adding direct URL information +License-Expression: Apache-2.0 +License-File: LICENSE +Author: Layered Maintainer +Author-email: maintainer@example.com +Project-URL: Documentation, https://example.com/layered/docs +Classifier: License :: OSI Approved :: Apache Software License diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD new file mode 100644 index 00000000..89ccbfe6 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD @@ -0,0 +1,9 @@ +layered/plugins/__init__.py,sha256=hMd8TidtznWDaiA4biHZ04ZoVXcAc7z/p77bIdAsPyE=,53 +layered/plugins/plugin.py,sha256=PBVqd9coVIzSBTQ2qdL5qxoK0fnsRZZ1DkhqnaVySPA=,87 +LICENSE,sha256=cXKP+wQk9Jyqh8mUi7nURl9jOOjojDqrabZ119S2EzM=,27 +layered-2.0.dist-info/METADATA,sha256=V09W93ILksWKLP8My6UatmScZ+5MCLiQ/5ieWzb585M=,346 +layered-2.0.dist-info/WHEEL,sha256=m8MHT7vQnqC5W8H/y4uJdEpx9ijH0jJGpuoaxRbcsQg=,79 +layered-2.0.dist-info/entry_points.txt,sha256=JYpkYczwozo6Ek7diDPgPj8ReYv5wTpaW0pFjL82bGU=,50 +layered-2.0.dist-info/INSTALLER,sha256=zuuue4knoyJ+UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg=,4 +layered-2.0.dist-info/direct_url.json,sha256=8NtnZQZq2S5tcEn+P5fH6/EpABJ9+Ha5aIq8Sn2szig=,189 +layered-2.0.dist-info/RECORD,, diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL new file mode 100644 index 00000000..e0965d4f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: pip 24.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/direct_url.json b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/direct_url.json new file mode 100644 index 00000000..8532fccc --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/direct_url.json @@ -0,0 +1,11 @@ +{ + "url": "https://git.example.com/layered", + "dir_info": { + "editable": true, + "subdirectory": "src/layered" + }, + "vcs_info": { + "vcs": "git", + "commit_id": "abc123" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt new file mode 100644 index 00000000..a25a2145 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[layered.hooks] +register=layered.plugins:register diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered/plugins/__init__.py b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered/plugins/__init__.py new file mode 100644 index 00000000..6ac87216 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered/plugins/__init__.py @@ -0,0 +1,3 @@ +from .plugin import register + +__all__ = ["register"] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered/plugins/plugin.py b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered/plugins/plugin.py new file mode 100644 index 00000000..98a88d6f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layer2/usr/lib/python3.11/site-packages/layered/plugins/plugin.py @@ -0,0 +1,5 @@ +from __future__ import annotations + + +def register() -> str: + return "layer2-plugin" diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json new file mode 100644 index 00000000..a9b5f3b8 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json @@ -0,0 +1,87 @@ +[ + { + "analyzerId": "python", + "componentKey": "purl::pkg:pypi/cache-pkg@1.2.3", + "purl": "pkg:pypi/cache-pkg@1.2.3", + "name": "Cache-Pkg", + "version": "1.2.3", + "type": "pypi", + "usedByEntrypoint": true, + "metadata": { + "classifier[0]": "Intended Audience :: Developers", + "classifier[1]": "License :: OSI Approved :: BSD License", + "classifier[2]": "Programming Language :: Python :: 3", + "classifiers": "Intended Audience :: Developers;License :: OSI Approved :: BSD License;Programming Language :: Python :: 3", + "distInfoPath": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info", + "entryPoints.console_scripts": "cache-tool=cache_pkg:main", + "installer": "pip", + "license": "BSD-3-Clause", + "license.classifier[0]": "License :: OSI Approved :: BSD License", + "license.file[0]": "LICENSE", + "name": "Cache-Pkg", + "normalizedName": "cache-pkg", + "projectUrl": "Source, https://example.com/cache-pkg", + "provenance": "dist-info", + "record.hashMismatches": "1", + "record.hashedEntries": "9", + "record.ioErrors": "0", + "record.missingFiles": "2", + "record.totalEntries": "12", + "record.unsupportedAlgorithms": "md5", + "requiresDist": "click", + "requiresPython": "\u003E=3.8", + "summary": "Cache test package for hashed RECORD coverage", + "version": "1.2.3", + "wheel.generator": "pip 24.0", + "wheel.rootIsPurelib": "true", + "wheel.tags": "py3-none-any", + "wheel.version": "1.0" + }, + "evidence": [ + { + "kind": "derived", + "source": "RECORD", + "locator": "../etc/passwd", + "value": "outside-root" + }, + { + "kind": "derived", + "source": "RECORD", + "locator": "lib/python3.11/site-packages/cache_pkg/LICENSE", + "value": "sha256 mismatch expected=pdUY6NGoyWCpcK8ThpBPUIVXbLvU9PzP8lsXEVEnFd0= actual=pdUY6NGoyWCpcK8ThpBPUIVXbLvU9PzP8lsXEVEnFdk=", + "sha256": "pdUY6NGoyWCpcK8ThpBPUIVXbLvU9PzP8lsXEVEnFdk=" + }, + { + "kind": "derived", + "source": "RECORD", + "locator": "lib/python3.11/site-packages/cache_pkg/missing/data.json", + "value": "missing" + }, + { + "kind": "file", + "source": "INSTALLER", + "locator": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/INSTALLER" + }, + { + "kind": "file", + "source": "METADATA", + "locator": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/METADATA" + }, + { + "kind": "file", + "source": "RECORD", + "locator": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/RECORD" + }, + { + "kind": "file", + "source": "WHEEL", + "locator": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/WHEEL" + }, + { + "kind": "file", + "source": "entry_points.txt", + "locator": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/entry_points.txt" + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.data/scripts/cache-tool b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.data/scripts/cache-tool new file mode 100644 index 00000000..30a18f13 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.data/scripts/cache-tool @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +from cache_pkg import main + +if __name__ == "__main__": + main() diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/INSTALLER b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/INSTALLER new file mode 100644 index 00000000..a1b589e3 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/METADATA b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/METADATA new file mode 100644 index 00000000..9a2fcee2 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/METADATA @@ -0,0 +1,12 @@ +Metadata-Version: 2.1 +Name: Cache-Pkg +Version: 1.2.3 +Summary: Cache test package for hashed RECORD coverage +License: BSD-3-Clause +License-File: LICENSE +Classifier: Programming Language :: Python :: 3 +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Requires-Python: >=3.8 +Requires-Dist: click +Project-URL: Source, https://example.com/cache-pkg diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/RECORD b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/RECORD new file mode 100644 index 00000000..af1ba92b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/RECORD @@ -0,0 +1,12 @@ +cache_pkg/__init__.py,sha256=iw2XGXcGU2Si1KAQ7o82tPSKxwEFc6UNZfITpGPh7mM=,189 +cache_pkg/data/config.json,sha256=Oa/wlgi1qJHC93RF5vwoOFffyviGtm5ccL7lrI0gkeY=,49 +cache_pkg/LICENSE,sha256=pdUY6NGoyWCpcK8ThpBPUIVXbLvU9PzP8lsXEVEnFd0=,73 +cache_pkg/md5only.txt,md5=Zm9v,4 +cache_pkg-1.2.3.data/scripts/cache-tool,sha256=2rsv/gnYOtlJZCy75Wz0rCADxYPnQAkyKvNbuoquZQ4=,89 +cache_pkg-1.2.3.dist-info/METADATA,sha256=DXPSItxOR1kPkVzjyq8F50jf8FOR9brSs/TGcZmcEHo=,390 +cache_pkg-1.2.3.dist-info/WHEEL,sha256=m8MHT7vQnqC5W8H/y4uJdEpx9ijH0jJGpuoaxRbcsQg=,79 +cache_pkg-1.2.3.dist-info/entry_points.txt,sha256=S1tGoBGlzWL6jeECCTw1pZP09HM8voEm/qQ7DtOHPyc=,44 +cache_pkg-1.2.3.dist-info/INSTALLER,sha256=zuuue4knoyJ+UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg=,4 +cache_pkg-1.2.3.dist-info/RECORD,, +cache_pkg/missing/data.json,sha256=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,12 +../../../../etc/passwd,, diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/WHEEL b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/WHEEL new file mode 100644 index 00000000..e0965d4f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: pip 24.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/entry_points.txt b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/entry_points.txt new file mode 100644 index 00000000..7cd4db1d --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +cache-tool=cache_pkg:main diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/LICENSE b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/LICENSE new file mode 100644 index 00000000..d9f6e52d --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/LICENSE @@ -0,0 +1,4 @@ +BSD 3-Clause License + +Copyright (c) 2025, StellaOps +All rights reserved. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/__init__.py b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/__init__.py new file mode 100644 index 00000000..acc51384 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/__init__.py @@ -0,0 +1,7 @@ +"""Cache fixture package for determinism tests.""" + +from .data import config # noqa: F401 + +def main() -> None: + """Entry point used by console script.""" + print("cache-pkg running") diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/data/config.json b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/data/config.json new file mode 100644 index 00000000..1538a394 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/data/config.json @@ -0,0 +1,4 @@ +{ + "cacheEnabled": true, + "ttlSeconds": 3600 +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/md5only.txt b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/md5only.txt new file mode 100644 index 00000000..257cc564 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/lib/python3.11/site-packages/cache_pkg/md5only.txt @@ -0,0 +1 @@ +foo diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json index 674cd49e..67ffc82c 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json @@ -10,19 +10,24 @@ "metadata": { "author": "Example Dev", "authorEmail": "dev@example.com", - "classifiers": "Programming Language :: Python :: 3;License :: OSI Approved :: Apache Software License", + "classifier[0]": "License :: OSI Approved :: Apache Software License", + "classifier[1]": "Programming Language :: Python :: 3", + "classifiers": "License :: OSI Approved :: Apache Software License;Programming Language :: Python :: 3", "distInfoPath": "lib/python3.11/site-packages/simple-1.0.0.dist-info", "editable": "true", "entryPoints.console_scripts": "simple-tool=simple.core:main", "homePage": "https://example.com/simple", "installer": "pip", "license": "Apache-2.0", + "license.classifier[0]": "License :: OSI Approved :: Apache Software License", "name": "simple", + "normalizedName": "simple", "projectUrl": "Source, https://example.com/simple/src", + "provenance": "dist-info", "record.hashMismatches": "0", - "record.hashedEntries": "9", + "record.hashedEntries": "8", "record.ioErrors": "0", - "record.missingFiles": "0", + "record.missingFiles": "1", "record.totalEntries": "10", "requiresDist": "requests (\u003E=2.0)", "requiresPython": "\u003E=3.9", @@ -38,11 +43,27 @@ "wheel.version": "1.0" }, "evidence": [ + { + "kind": "derived", + "source": "RECORD", + "locator": "bin/simple-tool", + "value": "missing" + }, + { + "kind": "file", + "source": "INSTALLER", + "locator": "lib/python3.11/site-packages/simple-1.0.0.dist-info/INSTALLER" + }, { "kind": "file", "source": "METADATA", "locator": "lib/python3.11/site-packages/simple-1.0.0.dist-info/METADATA" }, + { + "kind": "file", + "source": "RECORD", + "locator": "lib/python3.11/site-packages/simple-1.0.0.dist-info/RECORD" + }, { "kind": "file", "source": "WHEEL", @@ -61,4 +82,4 @@ } ] } -] \ No newline at end of file +] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs index 13e74dad..45eb6407 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs @@ -30,4 +30,54 @@ public sealed class PythonLanguageAnalyzerTests cancellationToken, usageHints); } + + [Fact] + public async Task PipCacheFixtureProducesDeterministicOutputAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "python", "pip-cache"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var usageHints = new LanguageUsageHints(new[] + { + Path.Combine(fixturePath, "lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool") + }); + + var analyzers = new ILanguageAnalyzer[] + { + new PythonLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken, + usageHints); + } + + [Fact] + public async Task LayeredEditableFixtureMergesAcrossLayersAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "python", "layered-editable"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var usageHints = new LanguageUsageHints(new[] + { + Path.Combine(fixturePath, "layer1", "usr", "bin", "layered-cli") + }); + + var analyzers = new ILanguageAnalyzer[] + { + new PythonLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken, + usageHints); + } } diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionLoader.cs b/src/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionLoader.cs index fa897240..8d5c1b51 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionLoader.cs +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionLoader.cs @@ -45,22 +45,97 @@ internal static class PythonDistributionLoader AddFileEvidence(context, metadataPath, "METADATA", evidenceEntries); AddFileEvidence(context, wheelPath, "WHEEL", evidenceEntries); AddFileEvidence(context, entryPointsPath, "entry_points.txt", evidenceEntries); + AddFileEvidence(context, installerPath, "INSTALLER", evidenceEntries); + AddFileEvidence(context, recordPath, "RECORD", evidenceEntries); AppendMetadata(metadataEntries, "distInfoPath", PythonPathHelper.NormalizeRelative(context, distInfoPath)); AppendMetadata(metadataEntries, "name", trimmedName); AppendMetadata(metadataEntries, "version", trimmedVersion); + AppendMetadata(metadataEntries, "normalizedName", normalizedName); AppendMetadata(metadataEntries, "summary", metadataDocument.GetFirst("Summary")); AppendMetadata(metadataEntries, "license", metadataDocument.GetFirst("License")); + AppendMetadata(metadataEntries, "licenseExpression", metadataDocument.GetFirst("License-Expression")); AppendMetadata(metadataEntries, "homePage", metadataDocument.GetFirst("Home-page")); AppendMetadata(metadataEntries, "author", metadataDocument.GetFirst("Author")); AppendMetadata(metadataEntries, "authorEmail", metadataDocument.GetFirst("Author-email")); AppendMetadata(metadataEntries, "projectUrl", metadataDocument.GetFirst("Project-URL")); AppendMetadata(metadataEntries, "requiresPython", metadataDocument.GetFirst("Requires-Python")); + var licenseFiles = metadataDocument.GetAll("License-File"); + if (licenseFiles.Count > 0) + { + var packageRoot = ResolvePackageRoot(distInfoPath); + var licenseIndex = 0; + var seenLicensePaths = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var licenseFile in licenseFiles) + { + if (string.IsNullOrWhiteSpace(licenseFile)) + { + continue; + } + + var trimmed = licenseFile.Trim(); + var resolved = TryResolvePackagePath(packageRoot, trimmed); + string metadataValue; + string? evidenceLocator = null; + + if (!string.IsNullOrEmpty(resolved) && File.Exists(resolved)) + { + metadataValue = PythonPathHelper.NormalizeRelative(context, resolved); + evidenceLocator = metadataValue; + } + else + { + metadataValue = trimmed; + } + + if (metadataValue.Length == 0 || !seenLicensePaths.Add(metadataValue)) + { + continue; + } + + AppendMetadata(metadataEntries, $"license.file[{licenseIndex}]", metadataValue); + licenseIndex++; + + if (!string.IsNullOrEmpty(evidenceLocator)) + { + evidenceEntries.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.File, + "license", + evidenceLocator, + Value: null, + Sha256: null)); + } + } + } + var classifiers = metadataDocument.GetAll("Classifier"); if (classifiers.Count > 0) { - AppendMetadata(metadataEntries, "classifiers", string.Join(';', classifiers)); + var orderedClassifiers = classifiers + .Where(static classifier => !string.IsNullOrWhiteSpace(classifier)) + .Select(static classifier => classifier.Trim()) + .OrderBy(static classifier => classifier, StringComparer.Ordinal) + .ToArray(); + + if (orderedClassifiers.Length > 0) + { + AppendMetadata(metadataEntries, "classifiers", string.Join(';', orderedClassifiers)); + + var licenseClassifierIndex = 0; + for (var index = 0; index < orderedClassifiers.Length; index++) + { + var classifier = orderedClassifiers[index]; + AppendMetadata(metadataEntries, $"classifier[{index}]", classifier); + + if (classifier.StartsWith("License ::", StringComparison.OrdinalIgnoreCase)) + { + AppendMetadata(metadataEntries, $"license.classifier[{licenseClassifierIndex}]", classifier); + licenseClassifierIndex++; + } + } + } } var requiresDist = metadataDocument.GetAll("Requires-Dist"); @@ -125,6 +200,7 @@ internal static class PythonDistributionLoader evidenceEntries.AddRange(verification.Evidence); var usedByEntrypoint = verification.UsedByEntrypoint || EvaluateEntryPointUsage(context, distInfoPath, entryPoints); + AppendMetadata(metadataEntries, "provenance", "dist-info"); return new PythonDistribution( trimmedName, @@ -267,6 +343,24 @@ internal static class PythonDistributionLoader return builder.ToString(); } + private static string ResolvePackageRoot(string distInfoPath) + { + var parent = Directory.GetParent(distInfoPath); + return parent?.FullName ?? distInfoPath; + } + + private static string? TryResolvePackagePath(string basePath, string relativePath) + { + try + { + return Path.GetFullPath(Path.Combine(basePath, relativePath)); + } + catch + { + return null; + } + } + private static async Task ReadSingleLineAsync(string path, CancellationToken cancellationToken) { if (!File.Exists(path)) @@ -769,6 +863,11 @@ internal static class PythonRecordVerifier var entryPath = entry.Path.Replace('/', Path.DirectorySeparatorChar); var fullPath = Path.GetFullPath(Path.Combine(parent, entryPath)); + if (context.UsageHints.IsPathUsed(fullPath)) + { + usedByEntrypoint = true; + } + if (!fullPath.StartsWith(root, StringComparison.Ordinal)) { missing++; @@ -793,11 +892,6 @@ internal static class PythonRecordVerifier continue; } - if (context.UsageHints.IsPathUsed(fullPath)) - { - usedByEntrypoint = true; - } - if (string.IsNullOrWhiteSpace(entry.HashAlgorithm) || string.IsNullOrWhiteSpace(entry.HashValue)) { continue; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md index 0144765d..4d588ba7 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md @@ -5,6 +5,6 @@ | 1 | SCANNER-ANALYZERS-LANG-10-303A | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-307 | STREAM-based parser for `*.dist-info` (`METADATA`, `WHEEL`, `entry_points.txt`) with normalization + evidence capture. | Parser handles CPython 3.8–3.12 metadata variations; fixtures confirm canonical ordering and UTF-8 handling. | | 2 | SCANNER-ANALYZERS-LANG-10-303B | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-303A | RECORD hash verifier with chunked hashing, Zip64 support, and mismatch diagnostics. | Verifier processes 5 GB RECORD fixture without allocations >2 MB; mismatches produce deterministic evidence records. | | 3 | SCANNER-ANALYZERS-LANG-10-303C | DONE (2025-10-21) | SCANNER-ANALYZERS-LANG-10-303B | Editable install + pip cache detection; integrate EntryTrace hints for runtime usage flags. | Editable installs resolved to source path; usage flags propagated; regression tests cover mixed editable + wheel installs. | -| 4 | SCANNER-ANALYZERS-LANG-10-307P | DOING (2025-10-23) | SCANNER-ANALYZERS-LANG-10-303C | Shared helper integration (license metadata, quiet provenance, component merging). | Shared helpers reused; analyzer-specific metadata minimal; deterministic merge tests pass. | -| 5 | SCANNER-ANALYZERS-LANG-10-308P | TODO | SCANNER-ANALYZERS-LANG-10-307P | Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. | Fixtures under `Fixtures/lang/python/`; determinism CI guard; benchmark CSV added with threshold alerts. | -| 6 | SCANNER-ANALYZERS-LANG-10-309P | TODO | SCANNER-ANALYZERS-LANG-10-308P | Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer; Offline Kit doc updated. | +| 4 | SCANNER-ANALYZERS-LANG-10-307P | DONE (2025-10-23) | SCANNER-ANALYZERS-LANG-10-303C | Shared helper integration (license metadata, quiet provenance, component merging). | Shared helpers reused; analyzer-specific metadata minimal; deterministic merge tests pass. | +| 5 | SCANNER-ANALYZERS-LANG-10-308P | DONE (2025-10-23) | SCANNER-ANALYZERS-LANG-10-307P | Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. | Fixtures under `Fixtures/lang/python/`; determinism CI guard; benchmark CSV added with threshold alerts. | +| 6 | SCANNER-ANALYZERS-LANG-10-309P | DONE (2025-10-23) | SCANNER-ANALYZERS-LANG-10-308P | Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer; Offline Kit doc updated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md index aa7522c4..02250523 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md +++ b/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md @@ -5,6 +5,6 @@ | 1 | SCANNER-ANALYZERS-LANG-10-306A | DONE (2025-10-22) | SCANNER-ANALYZERS-LANG-10-307 | Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. | Fixtures confirm crate attribution ≥85 % coverage; metadata normalized; evidence includes path + hash. | | 2 | SCANNER-ANALYZERS-LANG-10-306B | DONE (2025-10-22) | SCANNER-ANALYZERS-LANG-10-306A | Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. | Heuristic output flagged as `heuristic`; regression tests ensure no false “observed” classifications. | | 3 | SCANNER-ANALYZERS-LANG-10-306C | DONE (2025-10-22) | SCANNER-ANALYZERS-LANG-10-306B | Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. | Fallback path deterministic; shared helpers reused; tests verify consistent hashing. | -| 4 | SCANNER-ANALYZERS-LANG-10-307R | TODO | SCANNER-ANALYZERS-LANG-10-306C | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | Analyzer uses shared utilities; concurrency tests pass; no race conditions. | +| 4 | SCANNER-ANALYZERS-LANG-10-307R | DOING (2025-10-23) | SCANNER-ANALYZERS-LANG-10-306C | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | Analyzer uses shared utilities; concurrency tests pass; no race conditions. | | 5 | SCANNER-ANALYZERS-LANG-10-308R | TODO | SCANNER-ANALYZERS-LANG-10-307R | Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. | Fixtures `Fixtures/lang/rust/` committed; determinism guard; benchmark shows ≥15 % better coverage vs competitor. | | 6 | SCANNER-ANALYZERS-LANG-10-309R | TODO | SCANNER-ANALYZERS-LANG-10-308R | Package plug-in manifest + Offline Kit documentation; ensure Worker integration. | Manifest copied; Worker loads analyzer; Offline Kit doc updated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md b/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md index 3d325471..c9cc7523 100644 --- a/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md +++ b/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md @@ -37,6 +37,7 @@ All sprints below assume prerequisites from SP10-G2 (core scaffolding + Java ana - Usage hint propagation tests (EntryTrace → analyzer → SBOM). - Metrics counters (`scanner_analyzer_python_components_total`) documented. - **Progress (2025-10-21):** Python analyzer landed; Tasks 10-303A/B/C are DONE with dist-info parsing, RECORD verification, editable install detection, and deterministic `simple-venv` fixture + benchmark hooks recorded. +- **Progress (2025-10-23):** Closed Tasks 10-307P/308P/309P – Python analyzer now emits quiet-provenance metadata via shared helpers, determinism harness covers `simple-venv`, `pip-cache`, and `layered-editable` fixtures, bench suite reports hash throughput (`python/hash-throughput-20251023.csv`), and Offline Kit docs list the Python plug-in in the language bundle manifest. ## Sprint LA3 — Go Analyzer & Build Info Synthesis (Tasks 10-304, 10-307, 10-308, 10-309 subset) - **Scope:** Extract Go build metadata from `.note.go.buildid`, embedded module info, and fallback to `bin:{sha256}`; surface VCS provenance. diff --git a/src/StellaOps.Scanner.Emit/TASKS.md b/src/StellaOps.Scanner.Emit/TASKS.md index b0e36c10..7f23d275 100644 --- a/src/StellaOps.Scanner.Emit/TASKS.md +++ b/src/StellaOps.Scanner.Emit/TASKS.md @@ -8,5 +8,5 @@ | SCANNER-EMIT-10-604 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-602 | Package artifacts for export + attestation (naming, compression, manifests). | Export pipeline produces deterministic file paths/hashes; integration test with storage passes. | | SCANNER-EMIT-10-605 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-603 | Emit BOM-Index sidecar schema/fixtures (`bom-index@1`) and note CRITICAL PATH for Scheduler. | Schema + fixtures in docs/artifacts/bom-index; tests `BOMIndexGoldenIsStable` green. | | SCANNER-EMIT-10-606 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-605 | Integrate EntryTrace usage flags into BOM-Index; document semantics. | Usage bits present in sidecar; integration tests with EntryTrace fixtures pass. | -| SCANNER-EMIT-17-701 | TODO | Emit Guild, Native Analyzer Guild | SCANNER-EMIT-10-602 | Record GNU build-id for ELF components and surface it in inventory/usage SBOM plus diff payloads with deterministic ordering. | Native analyzer emits buildId for every ELF executable/library, SBOM/diff fixtures updated with canonical `buildId` field, regression tests prove stability, docs call out debug-symbol lookup flow. | +| SCANNER-EMIT-17-701 | DOING (2025-10-23) | Emit Guild, Native Analyzer Guild | SCANNER-EMIT-10-602 | Record GNU build-id for ELF components and surface it in inventory/usage SBOM plus diff payloads with deterministic ordering. | Native analyzer emits buildId for every ELF executable/library, SBOM/diff fixtures updated with canonical `buildId` field, regression tests prove stability, docs call out debug-symbol lookup flow. | | SCANNER-EMIT-10-607 | DONE (2025-10-22) | Emit Guild | SCANNER-EMIT-10-604, POLICY-CORE-09-005 | Embed scoring inputs, confidence band, and `quietedBy` provenance into CycloneDX 1.6 and DSSE predicates; verify deterministic serialization. | SBOM/attestation fixtures include score, inputs, configVersion, quiet metadata; golden tests confirm canonical output. | diff --git a/src/StellaOps.Scanner.Storage.Tests/RustFsArtifactObjectStoreTests.cs b/src/StellaOps.Scanner.Storage.Tests/RustFsArtifactObjectStoreTests.cs new file mode 100644 index 00000000..6b583b10 --- /dev/null +++ b/src/StellaOps.Scanner.Storage.Tests/RustFsArtifactObjectStoreTests.cs @@ -0,0 +1,168 @@ +using System.Linq; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Storage; +using StellaOps.Scanner.Storage.ObjectStore; +using Xunit; + +namespace StellaOps.Scanner.Storage.Tests; + +public sealed class RustFsArtifactObjectStoreTests +{ + [Fact] + public async Task PutAsync_PreservesStreamAndSendsImmutableHeaders() + { + var handler = new RecordingHttpMessageHandler(); + handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.OK)); + var factory = new SingleHttpClientFactory(new HttpClient(handler) + { + BaseAddress = new Uri("https://rustfs.test/api/v1/"), + }); + + var options = Options.Create(new ScannerStorageOptions + { + ObjectStore = + { + Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs, + BucketName = "scanner-artifacts", + RustFs = + { + BaseUrl = "https://rustfs.test/api/v1/", + Timeout = TimeSpan.FromSeconds(10), + }, + }, + }); + + options.Value.ObjectStore.Headers["X-Custom-Header"] = "custom-value"; + + var store = new RustFsArtifactObjectStore(factory, options, NullLogger.Instance); + + var payload = new MemoryStream(System.Text.Encoding.UTF8.GetBytes("rustfs artifact payload")); + var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/layers/digest/file.bin", true, TimeSpan.FromHours(1)); + + await store.PutAsync(descriptor, payload, CancellationToken.None); + + Assert.True(payload.CanRead); + Assert.Equal(0, payload.Position); + + var request = Assert.Single(handler.CapturedRequests); + Assert.Equal(HttpMethod.Put, request.Method); + Assert.Equal("https://rustfs.test/api/v1/buckets/scanner-artifacts/objects/scanner/layers/digest/file.bin", request.RequestUri.ToString()); + Assert.Contains("X-Custom-Header", request.Headers.Keys); + Assert.Equal("custom-value", Assert.Single(request.Headers["X-Custom-Header"])); + Assert.Equal("true", Assert.Single(request.Headers["X-RustFS-Immutable"])); + Assert.Equal("3600", Assert.Single(request.Headers["X-RustFS-Retain-Seconds"])); + Assert.Equal("application/octet-stream", Assert.Single(request.Headers["Content-Type"])); + } + + [Fact] + public async Task GetAsync_ReturnsNullOnNotFound() + { + var handler = new RecordingHttpMessageHandler(); + handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound)); + var factory = new SingleHttpClientFactory(new HttpClient(handler) + { + BaseAddress = new Uri("https://rustfs.test/api/v1/"), + }); + + var options = Options.Create(new ScannerStorageOptions + { + ObjectStore = + { + Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs, + BucketName = "scanner-artifacts", + RustFs = + { + BaseUrl = "https://rustfs.test/api/v1/", + }, + }, + }); + + var store = new RustFsArtifactObjectStore(factory, options, NullLogger.Instance); + var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/indexes/digest/index.bin", false); + + var result = await store.GetAsync(descriptor, CancellationToken.None); + + Assert.Null(result); + var request = Assert.Single(handler.CapturedRequests); + Assert.Equal(HttpMethod.Get, request.Method); + } + + [Fact] + public async Task DeleteAsync_IgnoresNotFound() + { + var handler = new RecordingHttpMessageHandler(); + handler.Responses.Enqueue(new HttpResponseMessage(HttpStatusCode.NotFound)); + var factory = new SingleHttpClientFactory(new HttpClient(handler) + { + BaseAddress = new Uri("https://rustfs.test/api/v1/"), + }); + + var options = Options.Create(new ScannerStorageOptions + { + ObjectStore = + { + Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs, + BucketName = "scanner-artifacts", + RustFs = + { + BaseUrl = "https://rustfs.test/api/v1/", + }, + }, + }); + + var store = new RustFsArtifactObjectStore(factory, options, NullLogger.Instance); + var descriptor = new ArtifactObjectDescriptor("scanner-artifacts", "scanner/attest/digest/attest.bin", false); + + await store.DeleteAsync(descriptor, CancellationToken.None); + + var request = Assert.Single(handler.CapturedRequests); + Assert.Equal(HttpMethod.Delete, request.Method); + } + + private sealed record CapturedRequest(HttpMethod Method, Uri RequestUri, IReadOnlyDictionary Headers); + + private sealed class RecordingHttpMessageHandler : HttpMessageHandler + { + public Queue Responses { get; } = new(); + + public List CapturedRequests { get; } = new(); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var headerSnapshot = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var header in request.Headers) + { + headerSnapshot[header.Key] = header.Value.ToArray(); + } + + if (request.Content is not null) + { + foreach (var header in request.Content.Headers) + { + headerSnapshot[header.Key] = header.Value.ToArray(); + } + + // Materialize content to ensure downstream callers can inspect it. + _ = await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + } + + CapturedRequests.Add(new CapturedRequest(request.Method, request.RequestUri!, headerSnapshot)); + return Responses.Count > 0 ? Responses.Dequeue() : new HttpResponseMessage(HttpStatusCode.OK); + } + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } +} diff --git a/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs index 60002aa1..bec39fc0 100644 --- a/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs @@ -1,8 +1,10 @@ -using Amazon; -using Amazon.S3; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; +using System; +using System.Net.Http; +using Amazon; +using Amazon.S3; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Driver; @@ -57,16 +59,61 @@ public static class ServiceCollectionExtensions services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); - - services.TryAddSingleton(CreateAmazonS3Client); - services.TryAddSingleton(); - services.TryAddSingleton(); - } + + services.AddHttpClient(RustFsArtifactObjectStore.HttpClientName) + .ConfigureHttpClient((sp, client) => + { + var options = sp.GetRequiredService>().Value.ObjectStore; + if (!options.IsRustFsDriver()) + { + return; + } + + if (!Uri.TryCreate(options.RustFs.BaseUrl, UriKind.Absolute, out var baseUri)) + { + throw new InvalidOperationException("RustFS baseUrl must be a valid absolute URI."); + } + + client.BaseAddress = baseUri; + client.Timeout = options.RustFs.Timeout; + + foreach (var header in options.Headers) + { + client.DefaultRequestHeaders.TryAddWithoutValidation(header.Key, header.Value); + } + + if (!string.IsNullOrWhiteSpace(options.RustFs.ApiKeyHeader) + && !string.IsNullOrWhiteSpace(options.RustFs.ApiKey)) + { + client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFs.ApiKeyHeader, options.RustFs.ApiKey); + } + }) + .ConfigurePrimaryHttpMessageHandler(sp => + { + var options = sp.GetRequiredService>().Value.ObjectStore; + if (!options.IsRustFsDriver()) + { + return new HttpClientHandler(); + } + + var handler = new HttpClientHandler(); + if (options.RustFs.AllowInsecureTls) + { + handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; + } + + return handler; + }); + + services.TryAddSingleton(CreateAmazonS3Client); + services.TryAddSingleton(CreateArtifactObjectStore); + services.TryAddSingleton(); + } private static IMongoClient CreateMongoClient(IServiceProvider provider) { @@ -95,11 +142,11 @@ public static class ServiceCollectionExtensions return client.GetDatabase(databaseName); } - private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider) - { - var options = provider.GetRequiredService>().Value.ObjectStore; - var config = new AmazonS3Config - { + private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider) + { + var options = provider.GetRequiredService>().Value.ObjectStore; + var config = new AmazonS3Config + { RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region), ForcePathStyle = options.ForcePathStyle, }; @@ -108,7 +155,26 @@ public static class ServiceCollectionExtensions { config.ServiceURL = options.ServiceUrl; } - - return new AmazonS3Client(config); - } -} + + return new AmazonS3Client(config); + } + + private static IArtifactObjectStore CreateArtifactObjectStore(IServiceProvider provider) + { + var options = provider.GetRequiredService>(); + var objectStore = options.Value.ObjectStore; + + if (objectStore.IsRustFsDriver()) + { + return new RustFsArtifactObjectStore( + provider.GetRequiredService(), + options, + provider.GetRequiredService>()); + } + + return new S3ArtifactObjectStore( + provider.GetRequiredService(), + options, + provider.GetRequiredService>()); + } +} diff --git a/src/StellaOps.Scanner.Storage/ObjectStore/RustFsArtifactObjectStore.cs b/src/StellaOps.Scanner.Storage/ObjectStore/RustFsArtifactObjectStore.cs new file mode 100644 index 00000000..6c87c6df --- /dev/null +++ b/src/StellaOps.Scanner.Storage/ObjectStore/RustFsArtifactObjectStore.cs @@ -0,0 +1,237 @@ +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Scanner.Storage.ObjectStore; + +public sealed class RustFsArtifactObjectStore : IArtifactObjectStore +{ + internal const string HttpClientName = "scanner-storage-rustfs"; + + private const string ImmutableHeader = "X-RustFS-Immutable"; + private const string RetainSecondsHeader = "X-RustFS-Retain-Seconds"; + private static readonly MediaTypeHeaderValue OctetStream = new("application/octet-stream"); + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IOptions _options; + private readonly ILogger _logger; + + public RustFsArtifactObjectStore( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(content); + + var storeOptions = _options.Value.ObjectStore; + EnsureRustFsDriver(storeOptions); + + var client = _httpClientFactory.CreateClient(HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Put, BuildRequestUri(storeOptions, descriptor)) + { + Content = CreateHttpContent(content), + }; + + request.Content.Headers.ContentType = OctetStream; + ApplyHeaders(storeOptions, request, descriptor); + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException( + $"RustFS upload for {descriptor.Bucket}/{descriptor.Key} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}"); + } + + _logger.LogDebug("Uploaded scanner object {Bucket}/{Key} via RustFS", descriptor.Bucket, descriptor.Key); + } + + public async Task GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(descriptor); + + var storeOptions = _options.Value.ObjectStore; + EnsureRustFsDriver(storeOptions); + + var client = _httpClientFactory.CreateClient(HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, BuildRequestUri(storeOptions, descriptor)); + ApplyHeaders(storeOptions, request, descriptor); + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogDebug("RustFS object {Bucket}/{Key} not found", descriptor.Bucket, descriptor.Key); + return null; + } + + if (!response.IsSuccessStatusCode) + { + var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException( + $"RustFS download for {descriptor.Bucket}/{descriptor.Key} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}"); + } + + var buffer = new MemoryStream(); + if (response.Content is not null) + { + await response.Content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + } + + buffer.Position = 0; + return buffer; + } + + public async Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(descriptor); + + var storeOptions = _options.Value.ObjectStore; + EnsureRustFsDriver(storeOptions); + + var client = _httpClientFactory.CreateClient(HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Delete, BuildRequestUri(storeOptions, descriptor)); + ApplyHeaders(storeOptions, request, descriptor); + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogDebug("RustFS object {Bucket}/{Key} already absent", descriptor.Bucket, descriptor.Key); + return; + } + + if (!response.IsSuccessStatusCode) + { + var error = await ReadErrorAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException( + $"RustFS delete for {descriptor.Bucket}/{descriptor.Key} failed with status {(int)response.StatusCode} ({response.ReasonPhrase}). {error}"); + } + + _logger.LogDebug("Deleted scanner object {Bucket}/{Key} via RustFS", descriptor.Bucket, descriptor.Key); + } + + private static void EnsureRustFsDriver(ObjectStoreOptions options) + { + if (!options.IsRustFsDriver()) + { + throw new InvalidOperationException("RustFS object store invoked while driver is not set to rustfs."); + } + } + + private static async Task ReadErrorAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + if (response.Content is null) + { + return string.Empty; + } + + var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(text)) + { + return string.Empty; + } + + var trimmed = text.Trim(); + return trimmed.Length <= 512 ? trimmed : trimmed[..512]; + } + + private static HttpContent CreateHttpContent(Stream content) + { + if (content is MemoryStream memoryStream) + { + if (memoryStream.TryGetBuffer(out var segment)) + { + return new ByteArrayContent(segment.Array!, segment.Offset, segment.Count); + } + + return new ByteArrayContent(memoryStream.ToArray()); + } + + if (content.CanSeek) + { + var originalPosition = content.Position; + try + { + content.Position = 0; + using var duplicate = new MemoryStream(); + content.CopyTo(duplicate); + var bytes = duplicate.ToArray(); + return new ByteArrayContent(bytes); + } + finally + { + content.Position = originalPosition; + } + } + + using var buffer = new MemoryStream(); + content.CopyTo(buffer); + return new ByteArrayContent(buffer.ToArray()); + } + + private static Uri BuildRequestUri(ObjectStoreOptions options, ArtifactObjectDescriptor descriptor) + { + if (!Uri.TryCreate(options.RustFs.BaseUrl, UriKind.Absolute, out var baseUri)) + { + throw new InvalidOperationException("RustFS baseUrl is invalid."); + } + + var encodedBucket = Uri.EscapeDataString(descriptor.Bucket); + var encodedKey = EncodeKey(descriptor.Key); + var relativePath = new StringBuilder() + .Append("buckets/") + .Append(encodedBucket) + .Append("/objects/") + .Append(encodedKey) + .ToString(); + + return new Uri(baseUri, relativePath); + } + + private static string EncodeKey(string key) + { + if (string.IsNullOrWhiteSpace(key)) + { + return string.Empty; + } + + var segments = key.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return string.Join('/', segments.Select(Uri.EscapeDataString)); + } + + private void ApplyHeaders(ObjectStoreOptions options, HttpRequestMessage request, ArtifactObjectDescriptor descriptor) + { + if (!string.IsNullOrWhiteSpace(options.RustFs.ApiKeyHeader) + && !string.IsNullOrWhiteSpace(options.RustFs.ApiKey)) + { + request.Headers.TryAddWithoutValidation(options.RustFs.ApiKeyHeader, options.RustFs.ApiKey); + } + + foreach (var header in options.Headers) + { + request.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (descriptor.Immutable) + { + request.Headers.TryAddWithoutValidation(ImmutableHeader, "true"); + if (descriptor.RetainFor is { } retain && retain > TimeSpan.Zero) + { + var seconds = Math.Ceiling(retain.TotalSeconds); + request.Headers.TryAddWithoutValidation(RetainSecondsHeader, seconds.ToString(CultureInfo.InvariantCulture)); + } + } + } +} diff --git a/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs b/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs index ed77024e..706b816b 100644 --- a/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs +++ b/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs @@ -3,11 +3,18 @@ namespace StellaOps.Scanner.Storage; public static class ScannerStorageDefaults { public const string DefaultDatabaseName = "scanner"; - public const string DefaultBucketName = "stellaops"; - public const string DefaultRootPrefix = "scanner"; - - public static class Collections - { + public const string DefaultBucketName = "stellaops"; + public const string DefaultRootPrefix = "scanner"; + + public static class ObjectStoreProviders + { + public const string S3 = "s3"; + public const string Minio = "minio"; + public const string RustFs = "rustfs"; + } + + public static class Collections + { public const string Artifacts = "artifacts"; public const string Images = "images"; public const string Layers = "layers"; diff --git a/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs b/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs index 374acec1..4ef8c7c2 100644 --- a/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs +++ b/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs @@ -1,4 +1,6 @@ -using MongoDB.Driver; +using System; +using System.Collections.Generic; +using MongoDB.Driver; namespace StellaOps.Scanner.Storage; @@ -69,42 +71,120 @@ public sealed class MongoOptions } } -public sealed class ObjectStoreOptions -{ - public string Region { get; set; } = "us-east-1"; - - public string? ServiceUrl { get; set; } - = null; - - public string BucketName { get; set; } = ScannerStorageDefaults.DefaultBucketName; - - public string RootPrefix { get; set; } = ScannerStorageDefaults.DefaultRootPrefix; - - public bool ForcePathStyle { get; set; } = true; - - public bool EnableObjectLock { get; set; } = false; - - public TimeSpan? ComplianceRetention { get; set; } - = TimeSpan.FromDays(90); - - public void EnsureValid() - { - if (string.IsNullOrWhiteSpace(BucketName)) - { - throw new InvalidOperationException("Scanner storage bucket name cannot be empty."); - } - - if (string.IsNullOrWhiteSpace(RootPrefix)) - { - throw new InvalidOperationException("Scanner storage root prefix cannot be empty."); - } - - if (ComplianceRetention is { } retention && retention <= TimeSpan.Zero) - { - throw new InvalidOperationException("Compliance retention must be positive when specified."); - } - } -} +public sealed class ObjectStoreOptions +{ + private static readonly HashSet S3Drivers = new(StringComparer.OrdinalIgnoreCase) + { + ScannerStorageDefaults.ObjectStoreProviders.S3, + ScannerStorageDefaults.ObjectStoreProviders.Minio, + }; + + private static readonly HashSet RustFsDrivers = new(StringComparer.OrdinalIgnoreCase) + { + ScannerStorageDefaults.ObjectStoreProviders.RustFs, + }; + + public string Driver { get; set; } = ScannerStorageDefaults.ObjectStoreProviders.S3; + + public string Region { get; set; } = "us-east-1"; + + public string? ServiceUrl { get; set; } + = null; + + public string BucketName { get; set; } = ScannerStorageDefaults.DefaultBucketName; + + public string RootPrefix { get; set; } = ScannerStorageDefaults.DefaultRootPrefix; + + public bool ForcePathStyle { get; set; } = true; + + public bool EnableObjectLock { get; set; } = false; + + public TimeSpan? ComplianceRetention { get; set; } + = TimeSpan.FromDays(90); + + public IDictionary Headers { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public RustFsOptions RustFs { get; set; } = new(); + + public bool IsS3Driver() + => S3Drivers.Contains(Driver); + + public bool IsRustFsDriver() + => RustFsDrivers.Contains(Driver); + + public void EnsureValid() + { + if (!IsS3Driver() && !IsRustFsDriver()) + { + throw new InvalidOperationException($"Scanner storage object store driver '{Driver}' is not supported."); + } + + if (string.IsNullOrWhiteSpace(BucketName)) + { + throw new InvalidOperationException("Scanner storage bucket name cannot be empty."); + } + + if (string.IsNullOrWhiteSpace(RootPrefix)) + { + throw new InvalidOperationException("Scanner storage root prefix cannot be empty."); + } + + if (ComplianceRetention is { } retention && retention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Compliance retention must be positive when specified."); + } + + if (IsRustFsDriver()) + { + RustFs ??= new RustFsOptions(); + RustFs.EnsureValid(); + } + } +} + +public sealed class RustFsOptions +{ + public string BaseUrl { get; set; } = string.Empty; + + public bool AllowInsecureTls { get; set; } + = false; + + public string? ApiKey { get; set; } + = null; + + public string ApiKeyHeader { get; set; } = string.Empty; + + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(60); + + public void EnsureValid() + { + if (string.IsNullOrWhiteSpace(BaseUrl)) + { + throw new InvalidOperationException("RustFS baseUrl must be configured."); + } + + if (!Uri.TryCreate(BaseUrl, UriKind.Absolute, out var uri)) + { + throw new InvalidOperationException("RustFS baseUrl must be an absolute URI."); + } + + if (!string.Equals(uri.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("RustFS baseUrl must use HTTP or HTTPS."); + } + + if (Timeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("RustFS timeout must be greater than zero."); + } + + if (!string.IsNullOrWhiteSpace(ApiKeyHeader) && string.IsNullOrWhiteSpace(ApiKey)) + { + throw new InvalidOperationException("RustFS API key header name requires a non-empty API key."); + } + } +} public sealed class DualWriteOptions { diff --git a/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj b/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj index 2acd5ed0..4ca2ca14 100644 --- a/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj +++ b/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj @@ -9,9 +9,10 @@ - - - - - - + + + + + + + diff --git a/src/StellaOps.Scanner.Storage/TASKS.md b/src/StellaOps.Scanner.Storage/TASKS.md index e3e6bd2e..8baf8227 100644 --- a/src/StellaOps.Scanner.Storage/TASKS.md +++ b/src/StellaOps.Scanner.Storage/TASKS.md @@ -3,6 +3,7 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | SCANNER-STORAGE-09-301 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-CORE-09-501 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | Collections created via bootstrapper; migrations recorded; indexes enforce uniqueness + TTL; majority read/write configured. | -| SCANNER-STORAGE-09-302 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-STORAGE-09-301 | MinIO layout, immutability policies, client abstraction, and configuration binding. | S3 client abstraction configurable via options; bucket/prefix defaults documented; immutability flags enforced with tests; config binding validated. | -| SCANNER-STORAGE-09-303 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-STORAGE-09-301, SCANNER-STORAGE-09-302 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | Dual-write service writes metadata + objects atomically; digest determinism covered by tests; TTL enforcement fixture passing. | -| SCANNER-STORAGE-09-304 | DONE (2025-10-19) | Scanner Storage Guild | SCANNER-STORAGE-09-303 | Adopt `TimeProvider` across storage timestamps for determinism. | Storage services/repositories use injected `TimeProvider`; tests cover timestamp determinism. | +| SCANNER-STORAGE-09-302 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-STORAGE-09-301 | MinIO layout, immutability policies, client abstraction, and configuration binding. | S3 client abstraction configurable via options; bucket/prefix defaults documented; immutability flags enforced with tests; config binding validated. | +| SCANNER-STORAGE-09-303 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-STORAGE-09-301, SCANNER-STORAGE-09-302 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | Dual-write service writes metadata + objects atomically; digest determinism covered by tests; TTL enforcement fixture passing. | +| SCANNER-STORAGE-09-304 | DONE (2025-10-19) | Scanner Storage Guild | SCANNER-STORAGE-09-303 | Adopt `TimeProvider` across storage timestamps for determinism. | Storage services/repositories use injected `TimeProvider`; tests cover timestamp determinism. | +| SCANNER-STORAGE-11-401 | DONE (2025-10-23) | Scanner Storage Guild | SCANNER-STORAGE-09-302 | Replace MinIO artifact store with RustFS driver, including migration tooling and configuration updates. | RustFS provider registered across Worker/WebService; data migration plan/tooling validated on staging; Helm/offline kit configs updated; regression tests cover RustFS paths with deterministic results. | diff --git a/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs b/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs index b6368503..37d117c8 100644 --- a/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs +++ b/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs @@ -17,13 +17,14 @@ internal sealed class ScannerApplicationFactory : WebApplicationFactory ["scanner:storage:dsn"] = string.Empty, ["scanner:queue:driver"] = "redis", ["scanner:queue:dsn"] = "redis://localhost:6379", - ["scanner:artifactStore:driver"] = "minio", - ["scanner:artifactStore:endpoint"] = "https://minio.local", - ["scanner:artifactStore:accessKey"] = "test-access", - ["scanner:artifactStore:secretKey"] = "test-secret", - ["scanner:artifactStore:bucket"] = "scanner-artifacts", - ["scanner:telemetry:minimumLogLevel"] = "Information", - ["scanner:telemetry:enableRequestLogging"] = "false", + ["scanner:artifactStore:driver"] = "rustfs", + ["scanner:artifactStore:endpoint"] = "https://rustfs.local/api/v1/", + ["scanner:artifactStore:accessKey"] = "test-access", + ["scanner:artifactStore:secretKey"] = "test-secret", + ["scanner:artifactStore:bucket"] = "scanner-artifacts", + ["scanner:artifactStore:timeoutSeconds"] = "30", + ["scanner:telemetry:minimumLogLevel"] = "Information", + ["scanner:telemetry:enableRequestLogging"] = "false", ["scanner:events:enabled"] = "false", ["scanner:features:enableSignedReports"] = "false" }; diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs index 0542fa99..a5178d8f 100644 --- a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs @@ -102,30 +102,39 @@ public sealed class ScannerWebServiceOptions public IDictionary DriverSettings { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); } - public sealed class ArtifactStoreOptions - { - public string Driver { get; set; } = "minio"; - - public string Endpoint { get; set; } = string.Empty; - - public bool UseTls { get; set; } = true; - - public string AccessKey { get; set; } = string.Empty; - - public string SecretKey { get; set; } = string.Empty; - - public string? SecretKeyFile { get; set; } - - public string Bucket { get; set; } = "scanner-artifacts"; - - public string? Region { get; set; } - - public bool EnableObjectLock { get; set; } = true; - - public int ObjectLockRetentionDays { get; set; } = 30; - - public IDictionary Headers { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); - } + public sealed class ArtifactStoreOptions + { + public string Driver { get; set; } = "rustfs"; + + public string Endpoint { get; set; } = string.Empty; + + public bool UseTls { get; set; } = true; + + public bool AllowInsecureTls { get; set; } + = false; + + public int TimeoutSeconds { get; set; } = 60; + + public string AccessKey { get; set; } = string.Empty; + + public string SecretKey { get; set; } = string.Empty; + + public string? SecretKeyFile { get; set; } + + public string Bucket { get; set; } = "scanner-artifacts"; + + public string? Region { get; set; } + + public bool EnableObjectLock { get; set; } = true; + + public int ObjectLockRetentionDays { get; set; } = 30; + + public string? ApiKey { get; set; } + + public string ApiKeyHeader { get; set; } = string.Empty; + + public IDictionary Headers { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } public sealed class FeatureFlagOptions { diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs index 9f70dbf6..175f15eb 100644 --- a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs @@ -23,10 +23,12 @@ public static class ScannerWebServiceOptionsValidator "rabbitmq" }; - private static readonly HashSet SupportedArtifactDrivers = new(StringComparer.OrdinalIgnoreCase) - { - "minio" - }; + private static readonly HashSet SupportedArtifactDrivers = new(StringComparer.OrdinalIgnoreCase) + { + "minio", + "s3", + "rustfs" + }; private static readonly HashSet SupportedEventDrivers = new(StringComparer.OrdinalIgnoreCase) { @@ -151,28 +153,53 @@ public static class ScannerWebServiceOptionsValidator } } - private static void ValidateArtifactStore(ScannerWebServiceOptions.ArtifactStoreOptions artifactStore) - { - if (!SupportedArtifactDrivers.Contains(artifactStore.Driver)) - { - throw new InvalidOperationException($"Unsupported artifact store driver '{artifactStore.Driver}'. Supported drivers: minio."); - } - - if (string.IsNullOrWhiteSpace(artifactStore.Endpoint)) - { - throw new InvalidOperationException("Artifact store endpoint must be configured."); - } - - if (string.IsNullOrWhiteSpace(artifactStore.Bucket)) - { - throw new InvalidOperationException("Artifact store bucket must be configured."); - } - - if (artifactStore.EnableObjectLock && artifactStore.ObjectLockRetentionDays <= 0) - { - throw new InvalidOperationException("Artifact store objectLockRetentionDays must be greater than zero when object lock is enabled."); - } - } + private static void ValidateArtifactStore(ScannerWebServiceOptions.ArtifactStoreOptions artifactStore) + { + if (!SupportedArtifactDrivers.Contains(artifactStore.Driver)) + { + throw new InvalidOperationException($"Unsupported artifact store driver '{artifactStore.Driver}'. Supported drivers: minio, s3, rustfs."); + } + + if (string.Equals(artifactStore.Driver, "rustfs", StringComparison.OrdinalIgnoreCase)) + { + if (string.IsNullOrWhiteSpace(artifactStore.Endpoint)) + { + throw new InvalidOperationException("Artifact store endpoint must be configured for RustFS."); + } + + if (!Uri.TryCreate(artifactStore.Endpoint, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Artifact store endpoint must be an absolute URI for RustFS."); + } + + if (artifactStore.TimeoutSeconds <= 0) + { + throw new InvalidOperationException("Artifact store timeoutSeconds must be greater than zero for RustFS."); + } + + if (string.IsNullOrWhiteSpace(artifactStore.Bucket)) + { + throw new InvalidOperationException("Artifact store bucket must be configured."); + } + + return; + } + + if (string.IsNullOrWhiteSpace(artifactStore.Endpoint)) + { + throw new InvalidOperationException("Artifact store endpoint must be configured."); + } + + if (string.IsNullOrWhiteSpace(artifactStore.Bucket)) + { + throw new InvalidOperationException("Artifact store bucket must be configured."); + } + + if (artifactStore.EnableObjectLock && artifactStore.ObjectLockRetentionDays <= 0) + { + throw new InvalidOperationException("Artifact store objectLockRetentionDays must be greater than zero when object lock is enabled."); + } + } private static void ValidateEvents(ScannerWebServiceOptions.EventsOptions eventsOptions) { diff --git a/src/StellaOps.Scanner.WebService/Program.cs b/src/StellaOps.Scanner.WebService/Program.cs index 790ef01c..b444ae55 100644 --- a/src/StellaOps.Scanner.WebService/Program.cs +++ b/src/StellaOps.Scanner.WebService/Program.cs @@ -108,9 +108,10 @@ builder.Services.AddScannerStorage(storageOptions => storageOptions.Mongo.UseMajorityReadConcern = true; storageOptions.Mongo.UseMajorityWriteConcern = true; - if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Endpoint)) + storageOptions.ObjectStore.Headers.Clear(); + foreach (var header in bootstrapOptions.ArtifactStore.Headers) { - storageOptions.ObjectStore.ServiceUrl = bootstrapOptions.ArtifactStore.Endpoint; + storageOptions.ObjectStore.Headers[header.Key] = header.Value; } if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Bucket)) @@ -118,16 +119,45 @@ builder.Services.AddScannerStorage(storageOptions => storageOptions.ObjectStore.BucketName = bootstrapOptions.ArtifactStore.Bucket; } - if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Region)) + var artifactDriver = bootstrapOptions.ArtifactStore.Driver?.Trim() ?? string.Empty; + if (string.Equals(artifactDriver, ScannerStorageDefaults.ObjectStoreProviders.RustFs, StringComparison.OrdinalIgnoreCase)) { - storageOptions.ObjectStore.Region = bootstrapOptions.ArtifactStore.Region; + storageOptions.ObjectStore.Driver = ScannerStorageDefaults.ObjectStoreProviders.RustFs; + storageOptions.ObjectStore.RustFs.BaseUrl = bootstrapOptions.ArtifactStore.Endpoint; + storageOptions.ObjectStore.RustFs.AllowInsecureTls = bootstrapOptions.ArtifactStore.AllowInsecureTls; + storageOptions.ObjectStore.RustFs.Timeout = TimeSpan.FromSeconds(Math.Max(1, bootstrapOptions.ArtifactStore.TimeoutSeconds)); + storageOptions.ObjectStore.RustFs.ApiKey = bootstrapOptions.ArtifactStore.ApiKey; + storageOptions.ObjectStore.RustFs.ApiKeyHeader = bootstrapOptions.ArtifactStore.ApiKeyHeader ?? string.Empty; + storageOptions.ObjectStore.EnableObjectLock = false; + storageOptions.ObjectStore.ComplianceRetention = null; } + else + { + var resolvedDriver = string.Equals(artifactDriver, ScannerStorageDefaults.ObjectStoreProviders.Minio, StringComparison.OrdinalIgnoreCase) + ? ScannerStorageDefaults.ObjectStoreProviders.Minio + : ScannerStorageDefaults.ObjectStoreProviders.S3; + storageOptions.ObjectStore.Driver = resolvedDriver; - storageOptions.ObjectStore.EnableObjectLock = bootstrapOptions.ArtifactStore.EnableObjectLock; - storageOptions.ObjectStore.ForcePathStyle = true; - storageOptions.ObjectStore.ComplianceRetention = bootstrapOptions.ArtifactStore.EnableObjectLock - ? TimeSpan.FromDays(Math.Max(1, bootstrapOptions.ArtifactStore.ObjectLockRetentionDays)) - : null; + if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Endpoint)) + { + storageOptions.ObjectStore.ServiceUrl = bootstrapOptions.ArtifactStore.Endpoint; + } + + if (!string.IsNullOrWhiteSpace(bootstrapOptions.ArtifactStore.Region)) + { + storageOptions.ObjectStore.Region = bootstrapOptions.ArtifactStore.Region; + } + + storageOptions.ObjectStore.EnableObjectLock = bootstrapOptions.ArtifactStore.EnableObjectLock; + storageOptions.ObjectStore.ForcePathStyle = true; + storageOptions.ObjectStore.ComplianceRetention = bootstrapOptions.ArtifactStore.EnableObjectLock + ? TimeSpan.FromDays(Math.Max(1, bootstrapOptions.ArtifactStore.ObjectLockRetentionDays)) + : null; + + storageOptions.ObjectStore.RustFs.ApiKey = null; + storageOptions.ObjectStore.RustFs.ApiKeyHeader = string.Empty; + storageOptions.ObjectStore.RustFs.BaseUrl = string.Empty; + } }); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/StellaOps.Web/TASKS.md b/src/StellaOps.Web/TASKS.md index f7903641..bf519f49 100644 --- a/src/StellaOps.Web/TASKS.md +++ b/src/StellaOps.Web/TASKS.md @@ -5,3 +5,4 @@ | WEB1.TRIVY-SETTINGS | DONE (2025-10-21) | UX Specialist, Angular Eng | Backend `/exporters/trivy-db` contract | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | ✅ Angular route `/concelier/trivy-db-settings` backed by `TrivyDbSettingsPageComponent` with reactive form; ✅ Overrides persisted via `ConcelierExporterClient` (`settings`/`run` endpoints); ✅ Manual run button saves current overrides then triggers export and surfaces run metadata. | | WEB1.TRIVY-SETTINGS-TESTS | DONE (2025-10-21) | UX Specialist, Angular Eng | WEB1.TRIVY-SETTINGS | **DONE (2025-10-21)** – Added headless Karma harness (`ng test --watch=false`) wired to ChromeHeadless/CI launcher, created `karma.conf.cjs`, updated npm scripts + docs with Chromium prerequisites so CI/offline runners can execute specs deterministically. | Angular CLI available (npm scripts chained), Karma suite for Trivy DB components passing locally and in CI, docs note required prerequisites. | | WEB1.DEPS-13-001 | DONE (2025-10-21) | UX Specialist, Angular Eng, DevEx | WEB1.TRIVY-SETTINGS-TESTS | Stabilise Angular workspace dependencies for CI/offline nodes: refresh `package-lock.json`, ensure Puppeteer/Chromium binaries optional, document deterministic install workflow. | `npm install` completes without manual intervention on air-gapped nodes, `npm test` headless run succeeds from clean checkout, README updated with lockfile + cache steps. | +| WEB-POLICY-FIXTURES-10-001 | DONE (2025-10-23) | Angular Eng | SAMPLES-13-004 | Wire policy preview/report doc fixtures into UI harness (test utility or Storybook substitute) with type bindings and validation guard so UI stays aligned with documented payloads. | JSON fixtures importable within Angular workspace, typed helpers exported for reuse, Karma spec validates critical fields (confidence band, unknown metrics, DSSE summary). | diff --git a/src/StellaOps.Web/src/app/core/api/policy-preview.models.ts b/src/StellaOps.Web/src/app/core/api/policy-preview.models.ts new file mode 100644 index 00000000..6a7a3cfa --- /dev/null +++ b/src/StellaOps.Web/src/app/core/api/policy-preview.models.ts @@ -0,0 +1,128 @@ +export interface PolicyPreviewRequestDto { + imageDigest: string; + findings: ReadonlyArray; + baseline?: ReadonlyArray; + policy?: PolicyPreviewPolicyDto; +} + +export interface PolicyPreviewPolicyDto { + content?: string; + format?: string; + actor?: string; + description?: string; +} + +export interface PolicyPreviewFindingDto { + id: string; + severity: string; + environment?: string; + source?: string; + vendor?: string; + license?: string; + image?: string; + repository?: string; + package?: string; + purl?: string; + cve?: string; + path?: string; + layerDigest?: string; + tags?: ReadonlyArray; +} + +export interface PolicyPreviewVerdictDto { + findingId: string; + status: string; + ruleName?: string | null; + ruleAction?: string | null; + notes?: string | null; + score?: number | null; + configVersion?: string | null; + inputs?: Readonly>; + quietedBy?: string | null; + quiet?: boolean | null; + unknownConfidence?: number | null; + confidenceBand?: string | null; + unknownAgeDays?: number | null; + sourceTrust?: string | null; + reachability?: string | null; +} + +export interface PolicyPreviewDiffDto { + findingId: string; + baseline: PolicyPreviewVerdictDto; + projected: PolicyPreviewVerdictDto; + changed: boolean; +} + +export interface PolicyPreviewIssueDto { + code: string; + message: string; + severity: string; + path: string; +} + +export interface PolicyPreviewResponseDto { + success: boolean; + policyDigest: string; + revisionId?: string | null; + changed: number; + diffs: ReadonlyArray; + issues: ReadonlyArray; +} + +export interface PolicyPreviewSample { + previewRequest: PolicyPreviewRequestDto; + previewResponse: PolicyPreviewResponseDto; +} + +export interface PolicyReportRequestDto { + imageDigest: string; + findings: ReadonlyArray; + baseline?: ReadonlyArray; +} + +export interface PolicyReportResponseDto { + report: PolicyReportDocumentDto; + dsse?: DsseEnvelopeDto | null; +} + +export interface PolicyReportDocumentDto { + reportId: string; + imageDigest: string; + generatedAt: string; + verdict: string; + policy: PolicyReportPolicyDto; + summary: PolicyReportSummaryDto; + verdicts: ReadonlyArray; + issues: ReadonlyArray; +} + +export interface PolicyReportPolicyDto { + revisionId?: string | null; + digest?: string | null; +} + +export interface PolicyReportSummaryDto { + total: number; + blocked: number; + warned: number; + ignored: number; + quieted: number; +} + +export interface DsseEnvelopeDto { + payloadType: string; + payload: string; + signatures: ReadonlyArray; +} + +export interface DsseSignatureDto { + keyId: string; + algorithm: string; + signature: string; +} + +export interface PolicyReportSample { + reportRequest: PolicyReportRequestDto; + reportResponse: PolicyReportResponseDto; +} diff --git a/src/StellaOps.Web/src/app/testing/policy-fixtures.spec.ts b/src/StellaOps.Web/src/app/testing/policy-fixtures.spec.ts new file mode 100644 index 00000000..3f37a8b6 --- /dev/null +++ b/src/StellaOps.Web/src/app/testing/policy-fixtures.spec.ts @@ -0,0 +1,46 @@ +import { getPolicyPreviewFixture, getPolicyReportFixture } from './policy-fixtures'; + +describe('policy fixtures', () => { + it('returns fresh clones for preview data', () => { + const first = getPolicyPreviewFixture(); + const second = getPolicyPreviewFixture(); + + expect(first).not.toBe(second); + expect(first.previewRequest).not.toBe(second.previewRequest); + expect(first.previewResponse.diffs).not.toBe(second.previewResponse.diffs); + }); + + it('exposes required policy preview fields', () => { + const { previewRequest, previewResponse } = getPolicyPreviewFixture(); + + expect(previewRequest.imageDigest).toMatch(/^sha256:[0-9a-f]{64}$/); + expect(Array.isArray(previewRequest.findings)).toBeTrue(); + expect(previewRequest.findings.length).toBeGreaterThan(0); + expect(previewResponse.success).toBeTrue(); + expect(previewResponse.policyDigest).toMatch(/^[0-9a-f]{64}$/); + expect(previewResponse.diffs.length).toBeGreaterThan(0); + + const diff = previewResponse.diffs[0]; + expect(diff.projected.confidenceBand).toBeDefined(); + expect(diff.projected.unknownConfidence).toBeGreaterThan(0); + expect(diff.projected.reachability).toBeDefined(); + }); + + it('aligns preview and report fixtures', () => { + const preview = getPolicyPreviewFixture(); + const report = getPolicyReportFixture(); + + expect(report.report.policy.digest).toEqual(preview.previewResponse.policyDigest); + expect(report.report.verdicts.length).toEqual(report.report.summary.total); + expect(report.report.verdicts.length).toBeGreaterThan(0); + expect(report.report.verdicts.some(v => v.confidenceBand != null)).toBeTrue(); + }); + + it('provides DSSE metadata for report fixture', () => { + const { reportResponse } = getPolicyReportFixture(); + + expect(reportResponse.dsse).toBeDefined(); + expect(reportResponse.dsse?.payloadType).toBe('application/vnd.stellaops.report+json'); + expect(reportResponse.dsse?.signatures?.length).toBeGreaterThan(0); + }); +}); diff --git a/src/StellaOps.Web/src/app/testing/policy-fixtures.ts b/src/StellaOps.Web/src/app/testing/policy-fixtures.ts new file mode 100644 index 00000000..db74d4b1 --- /dev/null +++ b/src/StellaOps.Web/src/app/testing/policy-fixtures.ts @@ -0,0 +1,21 @@ +import previewSample from '../../../../samples/policy/policy-preview-unknown.json'; +import reportSample from '../../../../samples/policy/policy-report-unknown.json'; +import { + PolicyPreviewSample, + PolicyReportSample, +} from '../core/api/policy-preview.models'; + +const previewFixture: PolicyPreviewSample = previewSample; +const reportFixture: PolicyReportSample = reportSample; + +export function getPolicyPreviewFixture(): PolicyPreviewSample { + return clone(previewFixture); +} + +export function getPolicyReportFixture(): PolicyReportSample { + return clone(reportFixture); +} + +function clone(value: T): T { + return JSON.parse(JSON.stringify(value)); +} diff --git a/src/StellaOps.Web/tsconfig.json b/src/StellaOps.Web/tsconfig.json index 16201006..c846ce4b 100644 --- a/src/StellaOps.Web/tsconfig.json +++ b/src/StellaOps.Web/tsconfig.json @@ -6,15 +6,16 @@ "strict": true, "noImplicitOverride": true, "noPropertyAccessFromIndexSignature": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "skipLibCheck": true, - "esModuleInterop": true, - "sourceMap": true, - "declaration": false, - "experimentalDecorators": true, - "moduleResolution": "node", - "importHelpers": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true, + "esModuleInterop": true, + "resolveJsonModule": true, + "sourceMap": true, + "declaration": false, + "experimentalDecorators": true, + "moduleResolution": "node", + "importHelpers": true, "target": "ES2022", "module": "ES2022", "useDefineForClassFields": false, diff --git a/tools/RustFsMigrator/Program.cs b/tools/RustFsMigrator/Program.cs new file mode 100644 index 00000000..f9aa7b05 --- /dev/null +++ b/tools/RustFsMigrator/Program.cs @@ -0,0 +1,286 @@ +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using System.Net.Http.Headers; + +var options = MigrationOptions.Parse(args); +if (options is null) +{ + MigrationOptions.PrintUsage(); + return 1; +} + +Console.WriteLine($"RustFS migrator starting (prefix: '{options.Prefix ?? ""}')"); +if (options.DryRun) +{ + Console.WriteLine("Dry-run enabled. No objects will be written to RustFS."); +} + +var s3Config = new AmazonS3Config +{ + ForcePathStyle = true, +}; + +if (!string.IsNullOrWhiteSpace(options.S3ServiceUrl)) +{ + s3Config.ServiceURL = options.S3ServiceUrl; + s3Config.UseHttp = options.S3ServiceUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase); +} + +if (!string.IsNullOrWhiteSpace(options.S3Region)) +{ + s3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.S3Region); +} + +using var s3Client = CreateS3Client(options, s3Config); +using var httpClient = CreateRustFsClient(options); + +var listRequest = new ListObjectsV2Request +{ + BucketName = options.S3Bucket, + Prefix = options.Prefix, + MaxKeys = 1000, +}; + +var migrated = 0; +var skipped = 0; + +do +{ + var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false); + foreach (var entry in response.S3Objects) + { + if (entry.Size == 0 && entry.Key.EndsWith('/')) + { + skipped++; + continue; + } + + Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)..."); + + if (options.DryRun) + { + migrated++; + continue; + } + + using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest + { + BucketName = options.S3Bucket, + Key = entry.Key, + }).ConfigureAwait(false); + + await using var memory = new MemoryStream(); + await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false); + memory.Position = 0; + + using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key)) + { + Content = new ByteArrayContent(memory.ToArray()), + }; + request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream"); + + if (options.Immutable) + { + request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true"); + } + + if (options.RetentionSeconds is { } retainSeconds) + { + request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString()); + } + + if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey)) + { + request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!); + } + + using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false); + if (!responseMessage.IsSuccessStatusCode) + { + var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false); + Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}"); + return 2; + } + + migrated++; + } + + listRequest.ContinuationToken = response.NextContinuationToken; +} while (!string.IsNullOrEmpty(listRequest.ContinuationToken)); + +Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers."); +return 0; + +static AmazonS3Client CreateS3Client(MigrationOptions options, AmazonS3Config config) +{ + if (!string.IsNullOrWhiteSpace(options.S3AccessKey) && !string.IsNullOrWhiteSpace(options.S3SecretKey)) + { + var credentials = new BasicAWSCredentials(options.S3AccessKey, options.S3SecretKey); + return new AmazonS3Client(credentials, config); + } + + return new AmazonS3Client(config); +} + +static HttpClient CreateRustFsClient(MigrationOptions options) +{ + var client = new HttpClient + { + BaseAddress = new Uri(options.RustFsEndpoint, UriKind.Absolute), + Timeout = TimeSpan.FromMinutes(5), + }; + + if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey)) + { + client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFsApiKeyHeader, options.RustFsApiKey); + } + + return client; +} + +static Uri BuildRustFsUri(MigrationOptions options, string key) +{ + var normalized = string.Join('/', key + .Split('/', StringSplitOptions.RemoveEmptyEntries) + .Select(Uri.EscapeDataString)); + + var builder = new UriBuilder(options.RustFsEndpoint) + { + Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}", + }; + + return builder.Uri; +} + +internal sealed record MigrationOptions +{ + public string S3Bucket { get; init; } = string.Empty; + + public string? S3ServiceUrl { get; init; } + = null; + + public string? S3Region { get; init; } + = null; + + public string? S3AccessKey { get; init; } + = null; + + public string? S3SecretKey { get; init; } + = null; + + public string RustFsEndpoint { get; init; } = string.Empty; + + public string RustFsBucket { get; init; } = string.Empty; + + public string? RustFsApiKeyHeader { get; init; } + = null; + + public string? RustFsApiKey { get; init; } + = null; + + public string? Prefix { get; init; } + = null; + + public bool Immutable { get; init; } + = false; + + public int? RetentionSeconds { get; init; } + = null; + + public bool DryRun { get; init; } + = false; + + public static MigrationOptions? Parse(string[] args) + { + var builder = new Dictionary(StringComparer.OrdinalIgnoreCase); + + for (var i = 0; i < args.Length; i++) + { + var key = args[i]; + if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase)) + { + var normalized = key[2..]; + if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase)) + { + builder[normalized] = "true"; + continue; + } + + if (i + 1 >= args.Length) + { + Console.Error.WriteLine($"Missing value for argument '{key}'."); + return null; + } + + builder[normalized] = args[++i]; + } + } + + if (!builder.TryGetValue("s3-bucket", out var bucket) || string.IsNullOrWhiteSpace(bucket)) + { + Console.Error.WriteLine("--s3-bucket is required."); + return null; + } + + if (!builder.TryGetValue("rustfs-endpoint", out var rustFsEndpoint) || string.IsNullOrWhiteSpace(rustFsEndpoint)) + { + Console.Error.WriteLine("--rustfs-endpoint is required."); + return null; + } + + if (!builder.TryGetValue("rustfs-bucket", out var rustFsBucket) || string.IsNullOrWhiteSpace(rustFsBucket)) + { + Console.Error.WriteLine("--rustfs-bucket is required."); + return null; + } + + int? retentionSeconds = null; + if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr)) + { + if (double.TryParse(retainStr, out var days) && days > 0) + { + retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60); + } + else + { + Console.Error.WriteLine("--retain-days must be a positive number."); + return null; + } + } + + return new MigrationOptions + { + S3Bucket = bucket, + S3ServiceUrl = builder.TryGetValue("s3-endpoint", out var s3Endpoint) ? s3Endpoint : null, + S3Region = builder.TryGetValue("s3-region", out var s3Region) ? s3Region : null, + S3AccessKey = builder.TryGetValue("s3-access-key", out var s3AccessKey) ? s3AccessKey : null, + S3SecretKey = builder.TryGetValue("s3-secret-key", out var s3SecretKey) ? s3SecretKey : null, + RustFsEndpoint = rustFsEndpoint!, + RustFsBucket = rustFsBucket!, + RustFsApiKeyHeader = builder.TryGetValue("rustfs-api-key-header", out var apiKeyHeader) ? apiKeyHeader : null, + RustFsApiKey = builder.TryGetValue("rustfs-api-key", out var apiKey) ? apiKey : null, + Prefix = builder.TryGetValue("prefix", out var prefix) ? prefix : null, + Immutable = builder.ContainsKey("immutable"), + RetentionSeconds = retentionSeconds, + DryRun = builder.ContainsKey("dry-run"), + }; + } + + public static void PrintUsage() + { + Console.WriteLine(@"Usage: dotnet run --project tools/RustFsMigrator -- \ + --s3-bucket \ + [--s3-endpoint http://minio:9000] \ + [--s3-region us-east-1] \ + [--s3-access-key key --s3-secret-key secret] \ + --rustfs-endpoint http://rustfs:8080 \ + --rustfs-bucket scanner-artifacts \ + [--rustfs-api-key-header X-API-Key --rustfs-api-key token] \ + [--prefix scanner/] \ + [--immutable] \ + [--retain-days 365] \ + [--dry-run]"); + } +} diff --git a/tools/RustFsMigrator/RustFsMigrator.csproj b/tools/RustFsMigrator/RustFsMigrator.csproj new file mode 100644 index 00000000..ed64291e --- /dev/null +++ b/tools/RustFsMigrator/RustFsMigrator.csproj @@ -0,0 +1,11 @@ + + + Exe + net10.0 + enable + enable + + + + +