From 3098e84de41e10cfdc116b3ab8facdf73d04c9c0 Mon Sep 17 00:00:00 2001 From: StellaOps Bot Date: Sun, 4 Jan 2026 14:54:52 +0200 Subject: [PATCH] save progress --- docs/benchmarks/scanner/deep-dives/secrets.md | 16 +- docs/full-features-list.md | 1641 +++++++++++++++++ ..._002_SCANNER_secret_leak_detection_core.md | 540 ++++++ ...0260104_003_SCANNER_secret_rule_bundles.md | 451 +++++ ...60104_004_POLICY_secret_dsl_integration.md | 543 ++++++ ..._20260104_005_AIRGAP_secret_offline_kit.md | 591 ++++++ ...T_20260104_001_BE_adaptive_noise_gating.md | 30 +- ...T_20260104_002_FE_noise_gating_delta_ui.md | 223 +++ docs/key-features.md | 2 + .../operations/secret-leak-detection.md | 160 +- .../operations/secrets-bundle-rotation.md | 298 +++ .../rules/secrets/sources/aws-access-key.json | 17 + .../rules/secrets/sources/aws-secret-key.json | 17 + .../secrets/sources/azure-storage-key.json | 17 + .../sources/database-connection-string.json | 16 + .../secrets/sources/datadog-api-key.json | 17 + .../secrets/sources/discord-bot-token.json | 17 + .../secrets/sources/docker-hub-token.json | 17 + .../secrets/sources/gcp-service-account.json | 17 + .../secrets/sources/generic-api-key.json | 15 + .../secrets/sources/generic-password.json | 16 + .../secrets/sources/github-app-token.json | 17 + offline/rules/secrets/sources/github-pat.json | 17 + offline/rules/secrets/sources/gitlab-pat.json | 17 + .../rules/secrets/sources/heroku-api-key.json | 17 + offline/rules/secrets/sources/jwt-secret.json | 17 + .../secrets/sources/mailchimp-api-key.json | 17 + offline/rules/secrets/sources/npm-token.json | 17 + .../rules/secrets/sources/nuget-api-key.json | 17 + .../rules/secrets/sources/private-key-ec.json | 17 + .../secrets/sources/private-key-generic.json | 17 + .../secrets/sources/private-key-openssh.json | 17 + .../secrets/sources/private-key-rsa.json | 17 + offline/rules/secrets/sources/pypi-token.json | 17 + .../secrets/sources/sendgrid-api-key.json | 17 + .../rules/secrets/sources/slack-token.json | 17 + .../rules/secrets/sources/slack-webhook.json | 17 + .../sources/stripe-restricted-key.json | 17 + .../secrets/sources/stripe-secret-key.json | 17 + .../secrets/sources/telegram-bot-token.json | 17 + .../rules/secrets/sources/twilio-api-key.json | 17 + .../Commands/Binary/BinaryCommandGroup.cs | 67 + .../Commands/Binary/BinaryCommandHandlers.cs | 236 +++ .../Commands/CommandHandlers.Secrets.cs | 429 +++++ .../Commands/SecretsCommandGroup.cs | 247 +++ src/Cli/StellaOps.Cli/StellaOps.Cli.csproj | 4 + src/Policy/StellaOps.Policy.Engine/AGENTS.md | 4 + .../Gates/StabilityDampingGate.cs | 384 ++++ .../Gates/StabilityDampingOptions.cs | 81 + .../Gates/StabilityDampingGateTests.cs | 347 ++++ .../Diagnostics/ScannerWorkerMetrics.cs | 56 + .../Options/ScannerWorkerOptions.cs | 45 + .../Processing/ScanStageNames.cs | 4 + .../Secrets/SecretsAnalyzerStageExecutor.cs | 235 +++ .../StellaOps.Scanner.Worker/Program.cs | 14 + .../StellaOps.Scanner.Worker.csproj | 1 + ...laOps.Scanner.Analyzers.Lang.Python.csproj | 21 + .../AGENTS.md | 128 ++ .../AssemblyInfo.cs | 3 + .../Bundles/BundleBuilder.cs | 345 ++++ .../Bundles/BundleManifest.cs | 151 ++ .../Bundles/BundleSigner.cs | 349 ++++ .../Bundles/BundleVerifier.cs | 527 ++++++ .../Bundles/RuleValidator.cs | 186 ++ .../Detectors/CompositeSecretDetector.cs | 139 ++ .../Detectors/EntropyCalculator.cs | 161 ++ .../Detectors/EntropyDetector.cs | 199 ++ .../Detectors/ISecretDetector.cs | 32 + .../Detectors/RegexDetector.cs | 137 ++ .../Detectors/SecretMatch.cs | 57 + .../Evidence/SecretFinding.cs | 52 + .../Evidence/SecretLeakEvidence.cs | 136 ++ .../GlobalUsings.cs | 10 + .../Masking/IPayloadMasker.cs | 23 + .../Masking/PayloadMasker.cs | 151 ++ .../Rules/IRulesetLoader.cs | 29 + .../Rules/RulesetLoader.cs | 227 +++ .../Rules/SecretConfidence.cs | 22 + .../Rules/SecretRule.cs | 191 ++ .../Rules/SecretRuleType.cs | 22 + .../Rules/SecretRuleset.cs | 115 ++ .../Rules/SecretSeverity.cs | 27 + .../SecretsAnalyzer.cs | 234 +++ .../SecretsAnalyzerHost.cs | 186 ++ .../SecretsAnalyzerOptions.cs | 118 ++ .../ServiceCollectionExtensions.cs | 90 + ...StellaOps.Scanner.Analyzers.Secrets.csproj | 32 + .../CallGraphServiceCollectionExtensions.cs | 2 + .../Contracts/ScanAnalysisKeys.cs | 4 + .../Bundles/BundleBuilderTests.cs | 264 +++ .../Bundles/BundleSignerTests.cs | 217 +++ .../Bundles/BundleVerifierTests.cs | 328 ++++ .../Bundles/RuleValidatorTests.cs | 228 +++ .../EntropyCalculatorTests.cs | 110 ++ .../PayloadMaskerTests.cs | 171 ++ .../RegexDetectorTests.cs | 235 +++ .../RulesetLoaderTests.cs | 348 ++++ .../SecretRuleTests.cs | 173 ++ .../SecretRulesetTests.cs | 208 +++ ...Ops.Scanner.Analyzers.Secrets.Tests.csproj | 32 + .../CallGraphDigestsTests.cs | 469 +++++ .../Extensions/VexLensEndpointExtensions.cs | 114 ++ src/VexLens/StellaOps.VexLens/AGENTS.md | 6 + .../Api/NoiseGatingApiModels.cs | 205 ++ .../StellaOps.VexLens/Delta/DeltaEntry.cs | 88 + .../StellaOps.VexLens/Delta/DeltaReport.cs | 183 ++ .../Delta/DeltaReportBuilder.cs | 347 ++++ .../StellaOps.VexLens/Delta/DeltaSection.cs | 86 + .../VexLensServiceCollectionExtensions.cs | 74 +- .../StellaOps.VexLens/NoiseGate/INoiseGate.cs | 310 ++++ .../NoiseGate/NoiseGateOptions.cs | 122 ++ .../NoiseGate/NoiseGateService.cs | 471 +++++ .../StellaOps.VexLens.csproj | 3 + .../Storage/IGatingStatisticsStore.cs | 99 + .../Storage/ISnapshotStore.cs | 96 + .../Storage/InMemoryGatingStatisticsStore.cs | 89 + .../Storage/InMemorySnapshotStore.cs | 83 + .../Delta/DeltaReportBuilderTests.cs | 364 ++++ .../NoiseGate/NoiseGateServiceTests.cs | 451 +++++ .../StellaOps.VexLens.Tests.csproj | 25 + .../src/app/core/api/noise-gating.client.ts | 302 +++ .../src/app/core/api/noise-gating.models.ts | 266 +++ .../app/features/triage/components/index.ts | 8 + .../delta-entry-card.component.ts | 344 ++++ .../gating-statistics-card.component.ts | 364 ++++ .../triage/components/noise-gating/index.ts | 10 + .../noise-gating-delta-report.component.ts | 393 ++++ .../noise-gating-summary-strip.component.ts | 305 +++ .../triage-canvas/triage-canvas.component.ts | 153 +- .../Deduplication/DeduplicatedEdge.cs | 138 ++ .../Deduplication/EdgeDeduplicator.cs | 137 ++ .../Deduplication/EdgeSemanticKey.cs | 134 ++ 132 files changed, 19783 insertions(+), 31 deletions(-) create mode 100644 docs/full-features-list.md create mode 100644 docs/implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md create mode 100644 docs/implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md create mode 100644 docs/implplan/SPRINT_20260104_004_POLICY_secret_dsl_integration.md create mode 100644 docs/implplan/SPRINT_20260104_005_AIRGAP_secret_offline_kit.md rename docs/implplan/{ => archived/2026-01-04-completed-sprints}/SPRINT_20260104_001_BE_adaptive_noise_gating.md (78%) create mode 100644 docs/implplan/archived/SPRINT_20260104_002_FE_noise_gating_delta_ui.md create mode 100644 docs/modules/scanner/operations/secrets-bundle-rotation.md create mode 100644 offline/rules/secrets/sources/aws-access-key.json create mode 100644 offline/rules/secrets/sources/aws-secret-key.json create mode 100644 offline/rules/secrets/sources/azure-storage-key.json create mode 100644 offline/rules/secrets/sources/database-connection-string.json create mode 100644 offline/rules/secrets/sources/datadog-api-key.json create mode 100644 offline/rules/secrets/sources/discord-bot-token.json create mode 100644 offline/rules/secrets/sources/docker-hub-token.json create mode 100644 offline/rules/secrets/sources/gcp-service-account.json create mode 100644 offline/rules/secrets/sources/generic-api-key.json create mode 100644 offline/rules/secrets/sources/generic-password.json create mode 100644 offline/rules/secrets/sources/github-app-token.json create mode 100644 offline/rules/secrets/sources/github-pat.json create mode 100644 offline/rules/secrets/sources/gitlab-pat.json create mode 100644 offline/rules/secrets/sources/heroku-api-key.json create mode 100644 offline/rules/secrets/sources/jwt-secret.json create mode 100644 offline/rules/secrets/sources/mailchimp-api-key.json create mode 100644 offline/rules/secrets/sources/npm-token.json create mode 100644 offline/rules/secrets/sources/nuget-api-key.json create mode 100644 offline/rules/secrets/sources/private-key-ec.json create mode 100644 offline/rules/secrets/sources/private-key-generic.json create mode 100644 offline/rules/secrets/sources/private-key-openssh.json create mode 100644 offline/rules/secrets/sources/private-key-rsa.json create mode 100644 offline/rules/secrets/sources/pypi-token.json create mode 100644 offline/rules/secrets/sources/sendgrid-api-key.json create mode 100644 offline/rules/secrets/sources/slack-token.json create mode 100644 offline/rules/secrets/sources/slack-webhook.json create mode 100644 offline/rules/secrets/sources/stripe-restricted-key.json create mode 100644 offline/rules/secrets/sources/stripe-secret-key.json create mode 100644 offline/rules/secrets/sources/telegram-bot-token.json create mode 100644 offline/rules/secrets/sources/twilio-api-key.json create mode 100644 src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs create mode 100644 src/Cli/StellaOps.Cli/Commands/SecretsCommandGroup.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingGate.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingOptions.cs create mode 100644 src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/StabilityDampingGateTests.cs create mode 100644 src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AGENTS.md create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AssemblyInfo.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleBuilder.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleManifest.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleSigner.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleVerifier.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/RuleValidator.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/CompositeSecretDetector.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyCalculator.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyDetector.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/ISecretDetector.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/RegexDetector.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/SecretMatch.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretFinding.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretLeakEvidence.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/GlobalUsings.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/IPayloadMasker.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/PayloadMasker.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/IRulesetLoader.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/RulesetLoader.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretConfidence.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRule.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleType.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleset.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretSeverity.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerHost.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerOptions.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/ServiceCollectionExtensions.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/StellaOps.Scanner.Analyzers.Secrets.csproj create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleBuilderTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleSignerTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleVerifierTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/RuleValidatorTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/EntropyCalculatorTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/PayloadMaskerTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RegexDetectorTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RulesetLoaderTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRuleTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRulesetTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/StellaOps.Scanner.Analyzers.Secrets.Tests.csproj create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.CallGraph.Tests/CallGraphDigestsTests.cs create mode 100644 src/VexLens/StellaOps.VexLens/Api/NoiseGatingApiModels.cs create mode 100644 src/VexLens/StellaOps.VexLens/Delta/DeltaEntry.cs create mode 100644 src/VexLens/StellaOps.VexLens/Delta/DeltaReport.cs create mode 100644 src/VexLens/StellaOps.VexLens/Delta/DeltaReportBuilder.cs create mode 100644 src/VexLens/StellaOps.VexLens/Delta/DeltaSection.cs create mode 100644 src/VexLens/StellaOps.VexLens/NoiseGate/INoiseGate.cs create mode 100644 src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateOptions.cs create mode 100644 src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateService.cs create mode 100644 src/VexLens/StellaOps.VexLens/Storage/IGatingStatisticsStore.cs create mode 100644 src/VexLens/StellaOps.VexLens/Storage/ISnapshotStore.cs create mode 100644 src/VexLens/StellaOps.VexLens/Storage/InMemoryGatingStatisticsStore.cs create mode 100644 src/VexLens/StellaOps.VexLens/Storage/InMemorySnapshotStore.cs create mode 100644 src/VexLens/__Tests/StellaOps.VexLens.Tests/Delta/DeltaReportBuilderTests.cs create mode 100644 src/VexLens/__Tests/StellaOps.VexLens.Tests/NoiseGate/NoiseGateServiceTests.cs create mode 100644 src/VexLens/__Tests/StellaOps.VexLens.Tests/StellaOps.VexLens.Tests.csproj create mode 100644 src/Web/StellaOps.Web/src/app/core/api/noise-gating.client.ts create mode 100644 src/Web/StellaOps.Web/src/app/core/api/noise-gating.models.ts create mode 100644 src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/delta-entry-card.component.ts create mode 100644 src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/gating-statistics-card.component.ts create mode 100644 src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/index.ts create mode 100644 src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-delta-report.component.ts create mode 100644 src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-summary-strip.component.ts create mode 100644 src/__Libraries/StellaOps.ReachGraph/Deduplication/DeduplicatedEdge.cs create mode 100644 src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeDeduplicator.cs create mode 100644 src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeSemanticKey.cs diff --git a/docs/benchmarks/scanner/deep-dives/secrets.md b/docs/benchmarks/scanner/deep-dives/secrets.md index 8eeb0070c..a66ae61ea 100644 --- a/docs/benchmarks/scanner/deep-dives/secrets.md +++ b/docs/benchmarks/scanner/deep-dives/secrets.md @@ -1,6 +1,10 @@ # Secret Handling & Leak Detection -## StellaOps approach (2025.11 release) +> **Implementation Status:** Secret leak detection (`StellaOps.Scanner.Analyzers.Secrets`) is **NOT YET IMPLEMENTED**. +> Only Surface.Secrets (operational credential management) is currently available. +> See implementation sprints: SPRINT_20260104_002 through SPRINT_20260104_005. + +## StellaOps approach (TARGET - 2026.01+ release) - Read the Policy/Security briefing: `../../modules/policy/secret-leak-detection-readiness.md`. - Operational runbook: `../../modules/scanner/operations/secret-leak-detection.md`. - Surface.Secrets continues to deliver operational credentials through secure handles (`docs/modules/scanner/design/surface-secrets.md`), with providers supporting Kubernetes Secrets, file bundles, and inline definitions validated by Surface.Validation. @@ -25,15 +29,15 @@ - Operators must rely on external tooling for leak detection while Grype focuses exclusively on vulnerability matching.[g1] ## Key differences -- **Purpose**: StellaOps now covers both operational secret retrieval *and* deterministic leak detection; Trivy and Snyk focus exclusively on leak detection while Grype omits it. -- **Workflow**: StellaOps performs leak detection in-line during scans with offline rule bundles and policy-aware outcomes; Trivy/Snyk rely on mutable rule packs or SaaS classifiers; Grype delegates to external tools. -- **Determinism**: StellaOps signs every bundle and records bundle IDs in explain traces; Trivy and Snyk update rules continuously (risking drift); Grype remains deterministic by not scanning. +- **Purpose**: StellaOps currently provides operational secret retrieval only; leak detection is **PLANNED** (see implementation sprints). Trivy and Snyk focus exclusively on leak detection while Grype omits it. +- **Workflow**: StellaOps will perform leak detection in-line during scans with offline rule bundles and policy-aware outcomes (when implemented); Trivy/Snyk rely on mutable rule packs or SaaS classifiers; Grype delegates to external tools. +- **Determinism**: StellaOps will sign every bundle and record bundle IDs in explain traces (when implemented); Trivy and Snyk update rules continuously (risking drift); Grype remains deterministic by not scanning. ### Detection technique comparison | Tool | Detection technique(s) | Merge / result handling | Notes | | --- | --- | --- | --- | -| **StellaOps (≤ 2025.10)** | `Surface.Secrets` providers fetch credentials at runtime; no leak scanning. | Secrets resolve to opaque handles stored in scan metadata; no SBOM entries emitted. | Deterministic secret retrieval only (legacy behaviour). | -| **StellaOps (2025.11+)** | `StellaOps.Scanner.Analyzers.Secrets` plug-in executes DSSE-signed rule bundles. | Findings inserted into `ScanAnalysisStore` as `secret.leak` evidence; Policy Engine merges with component context and lattice scores; CLI/export mask payloads. | Rule bundles ship offline, signatures verified locally; see operations runbook for rollout. | +| **StellaOps (≤ 2025.10)** | `Surface.Secrets` providers fetch credentials at runtime; no leak scanning. | Secrets resolve to opaque handles stored in scan metadata; no SBOM entries emitted. | Deterministic secret retrieval only. **This is the current implementation.** | +| **StellaOps (PLANNED)** | `StellaOps.Scanner.Analyzers.Secrets` plug-in executes DSSE-signed rule bundles. | Findings inserted into `ScanAnalysisStore` as `secret.leak` evidence; Policy Engine merges with component context and lattice scores; CLI/export mask payloads. | **NOT YET IMPLEMENTED** - See SPRINT_20260104_002 through SPRINT_20260104_005. | | **Trivy** | Regex + entropy detectors under `pkg/fanal/secret` (configurable via `trivy-secret.yaml`). | Detectors aggregate per file; results exported alongside vulnerability findings without provenance binding. | Ships built-in rule sets; users can add allow/block lists. | | **Snyk** | Snyk Code SaaS classifiers invoked by CLI plugin (`src/lib/plugins/sast`). | Source uploaded to SaaS; issues returned with severity + remediation; no offline merge with SBOM data. | Requires authenticated cloud access; rules evolve server-side. | | **Grype** | None (focuses on vulnerability matching). | — | Operators must integrate separate tooling for leak detection. | diff --git a/docs/full-features-list.md b/docs/full-features-list.md new file mode 100644 index 000000000..0fe8ad543 --- /dev/null +++ b/docs/full-features-list.md @@ -0,0 +1,1641 @@ +# Full Features List - Stella Ops + +> **Comprehensive catalog of every capability in the Stella Ops platform.** +> +> For quick capability cards with competitive differentiation, see [`key-features.md`](key-features.md). +> For tier-based availability (Free/Community/Enterprise), see [`04_FEATURE_MATRIX.md`](04_FEATURE_MATRIX.md). + +--- + +## Table of Contents + +1. [Core Platform Differentiators](#1-core-platform-differentiators) +2. [Container Image Scanning](#2-container-image-scanning) +3. [SBOM Capabilities](#3-sbom-capabilities) +4. [Language Analyzers](#4-language-analyzers) +5. [Vulnerability Detection](#5-vulnerability-detection) +6. [Advisory Sources](#6-advisory-sources) +7. [VEX Processing](#7-vex-processing) +8. [Reachability Analysis](#8-reachability-analysis) +9. [Binary Analysis](#9-binary-analysis) +10. [Policy Engine](#10-policy-engine) +11. [Attestation & Signing](#11-attestation--signing) +12. [Regional Cryptography](#12-regional-cryptography) +13. [Risk Scoring & Assessment](#13-risk-scoring--assessment) +14. [Evidence Management](#14-evidence-management) +15. [Determinism & Reproducibility](#15-determinism--reproducibility) +16. [CLI Features](#16-cli-features) +17. [Web UI Features](#17-web-ui-features) +18. [Offline & Air-Gap Operations](#18-offline--air-gap-operations) +19. [Deployment Options](#19-deployment-options) +20. [Authentication & Authorization](#20-authentication--authorization) +21. [Integrations & Notifications](#21-integrations--notifications) +22. [Observability & Telemetry](#22-observability--telemetry) +23. [Scheduling & Automation](#23-scheduling--automation) +24. [Version Comparison](#24-version-comparison) +25. [Database & Storage](#25-database--storage) +26. [API Capabilities](#26-api-capabilities) +27. [Support & Services](#27-support--services) + +--- + +## 1. Core Platform Differentiators + +These are the fundamental capabilities that distinguish Stella Ops from other vulnerability scanners. + +### 1.1 Decision Capsules + +- **Audit-grade evidence bundles** containing everything needed to reproduce and verify vulnerability decisions +- Content-addressed bundles with exact SBOM, frozen feed snapshots (with Merkle roots), policy version, lattice rules +- Evidence includes reachability proofs (static + runtime), VEX statements, binary fingerprints +- Outputs include verdicts, risk scores, remediation paths +- DSSE signatures over all components +- Six-month-later replay: `stella replay srm.yaml --assert-digest ` produces identical results + +### 1.2 Deterministic Replay + +- **Bit-for-bit reproducible scans** from frozen feeds and analyzer manifests +- Replay Manifest (SRM) captures exact analyzer inputs/outputs per layer +- Feed snapshots (NVD, KEV, EPSS, distro advisories) with content hashes +- Frozen analyzer versions and configurations +- Frozen policy rules and lattice state +- Random seeds for deterministic ordering + +### 1.3 VEX-First Decisioning (K4 Lattice Logic) + +- **Belnap K4 four-valued logic** (Unknown, True, False, Conflict) +- VEX as logical claims with trust weighting, not suppression files +- Conflicts are explicit state, not hidden +- Vendor + runtime + reachability merged with conflicts surfaced +- Unknown treated as first-class state with risk implications + +### 1.4 Signed Reachability Proofs + +- **Three-layer validation** with cryptographic binding +- Every reachability graph sealed with DSSE +- Optional edge-bundle attestations for contested paths +- Proves exploitability with exact call paths from entrypoint to vulnerable function + +### 1.5 Sovereign Offline Operation + +- **Full functionality without network** +- Air-gapped environments get identical results to connected +- Offline Update Kits bundle everything needed +- Epistemic parity (sealed, reproducible knowledge state) + +### 1.6 Smart-Diff (Semantic Risk Delta) + +- **Diff security meaning, not CVE counts** +- Compare reachability graphs, policy outcomes, and trust weights between releases +- Output like "Exploitability DECREASED by 67% despite +2 CVEs" +- Material change detection for informed decision-making + +### 1.7 Unknowns as First-Class State + +- **Explicit modeling of uncertainty** +- Hot/Warm/Cold/Resolved bands for uncertainty tracking +- Decay algorithms for uncertainty resolution +- Blast-radius containment +- Policy budgets ("fail if unknowns > N") + +--- + +## 2. Container Image Scanning + +### 2.1 Image Formats + +- OCI container images +- Docker images +- Container filesystem archives +- Rootfs directories +- Layer-by-layer analysis + +### 2.2 Scanning Modes + +- **Quick Mode**: Fast scan for basic vulnerabilities +- **Standard Mode**: Balanced scan with full vulnerability detection +- **Deep Mode**: Comprehensive analysis with reachability and binary analysis + +### 2.3 Base Image Detection + +- Automatic base image identification +- Base image layer separation +- Inherited vs. application-added package differentiation + +### 2.4 Layer-Aware Analysis + +- Per-layer package detection +- Layer change tracking +- Delta analysis between layers +- Content-addressed layer caching + +### 2.5 Registry Integration + +- Pull images by digest (content-addressed) +- Registry authentication support +- Private registry support +- Registry mirror support for offline operation + +### 2.6 Scan Performance + +- Delta-SBOM cache for warm scans < 1 second +- Concurrent scan workers (1/3/unlimited by tier) +- Content-addressed layer caching +- Incremental analysis for unchanged layers + +--- + +## 3. SBOM Capabilities + +### 3.1 SBOM Formats Supported + +- **CycloneDX 1.7** (primary output format) +- **CycloneDX 1.6** (backward compatible ingest) +- **SPDX 3.0.1** (full support) +- **SPDX-JSON** (ingest) +- **Trivy-JSON** (ingest) + +### 3.2 SBOM Generation + +- Automatic SBOM generation from container images +- Package extraction from all supported ecosystems +- Dependency relationship mapping +- Component metadata extraction +- License detection + +### 3.3 SBOM Ingestion + +- Auto-format detection +- Bring-Your-Own-SBOM (BYOS) support +- Third-party SBOM import +- Validation and normalization + +### 3.4 Delta-SBOM Engine + +- Content-addressed catalog +- Layer-aware ingestion +- Rescans only fetch new layers +- Warm scans < 1 second + +### 3.5 SBOM Diff + +- Semantic SBOM comparison +- Package addition/removal detection +- Version change tracking +- License change detection + +### 3.6 SBOM Lineage Ledger (Enterprise) + +- Full versioned SBOM history +- Lineage tracking across builds +- Traversal queries via Lineage API +- Audit trail for SBOM changes + +### 3.7 SBOM Service + +- Central SBOM storage and versioning +- Content-addressed storage +- SBOM deduplication +- Retention policies + +--- + +## 4. Language Analyzers + +### 4.1 .NET/C# Analyzer + +- NuGet package detection +- packages.config parsing +- .csproj/Directory.Build.props parsing +- .NET SDK version detection +- Framework dependency mapping +- Assembly metadata extraction + +### 4.2 Java Analyzer + +- Maven dependency resolution (pom.xml) +- Gradle build file parsing (build.gradle, build.gradle.kts) +- JAR/WAR/EAR analysis +- MANIFEST.MF parsing +- Java version detection +- Spring Boot dependency detection + +### 4.3 Go Analyzer + +- go.mod/go.sum parsing +- Go module dependency resolution +- Go version detection +- Vendor directory analysis +- Binary build info extraction + +### 4.4 Python Analyzer + +- requirements.txt parsing +- Pipfile/Pipfile.lock parsing +- pyproject.toml parsing +- setup.py analysis +- Poetry lockfile support +- Conda environment parsing +- pip freeze output parsing + +### 4.5 Node.js Analyzer + +- package.json/package-lock.json parsing +- yarn.lock parsing +- pnpm-lock.yaml parsing +- npm shrinkwrap support +- Node.js version detection +- Workspace/monorepo support + +### 4.6 Ruby Analyzer + +- Gemfile/Gemfile.lock parsing +- Ruby version detection +- Bundler version detection +- Gem specification parsing + +### 4.7 Bun Analyzer + +- bun.lockb parsing +- package.json processing +- Bun-specific dependency resolution + +### 4.8 Deno Analyzer + +- deno.json parsing +- Import map resolution +- URL-based dependency tracking +- deno.lock parsing + +### 4.9 PHP Analyzer + +- composer.json/composer.lock parsing +- PHP version detection +- Packagist dependency resolution + +### 4.10 Rust Analyzer + +- Cargo.toml/Cargo.lock parsing +- Rust edition detection +- Crates.io dependency resolution +- Build target analysis + +### 4.11 Native Binary Analyzer + +- ELF binary analysis (Linux) +- PE binary analysis (Windows) +- Mach-O binary analysis (macOS) +- Build-ID extraction +- Symbol table parsing +- Dynamic library dependency detection + +--- + +## 5. Vulnerability Detection + +### 5.1 CVE Matching + +- CVE lookup via local database +- Package-to-CVE mapping +- Version range matching +- PURL-based matching + +### 5.2 Vulnerability Scoring + +- CVSS v4.0 display +- CVSS v3.1 support +- CVSS v2.0 legacy support +- EPSS v4 probability scoring +- Priority band classification + +### 5.3 Exploitability Assessment + +- KEV (Known Exploited Vulnerabilities) flagging +- EPSS probability integration +- Reachability-aware prioritization +- VEX status consideration + +### 5.4 License Risk Detection (Planned) + +- License identification +- License compatibility analysis +- License risk scoring +- Copyleft detection + +--- + +## 6. Advisory Sources + +### 6.1 Primary Sources + +- **NVD** (National Vulnerability Database) +- **GHSA** (GitHub Security Advisories) +- **OSV** (Open Source Vulnerabilities) +- **KEV** (Known Exploited Vulnerabilities) +- **EPSS v4** (Exploit Prediction Scoring System) + +### 6.2 Distribution-Specific Sources + +- **Alpine SecDB** +- **Debian Security Tracker** +- **Ubuntu USN** (Ubuntu Security Notices) +- **RHEL/CentOS OVAL** (Community/Enterprise) + +### 6.3 Advisory Processing (Concelier) + +- Multi-source advisory ingestion +- Advisory normalization +- Duplicate detection +- Conflict resolution +- Advisory merge engine (Enterprise) +- Custom advisory connectors (Enterprise) + +### 6.4 Feed Management + +- Automated feed updates +- Feed mirroring for offline operation +- Feed snapshot versioning +- Content-addressed feed storage + +--- + +## 7. VEX Processing + +### 7.1 VEX Formats Supported + +- **OpenVEX** (primary format) +- **CycloneDX VEX** +- **CSAF VEX** (Community/Enterprise) + +### 7.2 VEX Ingestion (Excititor) + +- Multi-format VEX import +- VEX validation +- VEX normalization +- Statement extraction + +### 7.3 VEX Consensus Engine (VexLens) + +- Trust vector scoring (Precision/Coverage/Recency) +- Claim strength multipliers +- Freshness decay algorithms +- Conflict detection and penalty (K4 lattice logic) +- Multi-issuer statement aggregation + +### 7.4 Trust Weighting + +- Issuer trust scoring +- Statement freshness weighting +- Claim strength assessment +- Conflict penalty calculation + +### 7.5 VEX Conflict Resolution + +- K4 four-valued logic (Unknown/True/False/Conflict) +- Conflict surfacing (not hiding) +- Visual conflict resolution (VEX Conflict Studio UI) +- Deterministic outcome selection + +### 7.6 VEX Hub + +- VEX distribution and exchange +- Internal VEX network +- VEX statement sharing +- VEX propagation across supply chain + +### 7.7 Issuer Directory + +- Issuer trust registry +- CSAF publisher management +- Trust root configuration +- Issuer metadata storage + +### 7.8 Trust Calibration Service (Enterprise) + +- Organization-specific trust tuning +- Custom trust weightings +- Historical trust analysis + +--- + +## 8. Reachability Analysis + +### 8.1 Static Call Graph + +- Function-level call graph construction +- Cross-module call tracking +- Entry point identification +- Path enumeration + +### 8.2 Entrypoint Detection + +- 9+ framework types supported +- HTTP endpoints +- CLI entry points +- Event handlers +- Message consumers +- Scheduled tasks + +### 8.3 BFS Reachability + +- Breadth-first path search +- Shortest path calculation +- All paths enumeration +- Path filtering + +### 8.4 Three-Layer Reachability Proofs + +- **Layer 1 (Static)**: Call graph path from entrypoint to vulnerable function +- **Layer 2 (Binary)**: Compiled binary contains symbol with matching offset +- **Layer 3 (Runtime)**: eBPF probe confirms function execution + +### 8.5 Confidence Tiers + +- **Confirmed**: All three layers agree +- **Likely**: Static + binary agree; no runtime data +- **Present**: Package present; no reachability evidence +- **Unreachable**: Static analysis proves no path exists + +### 8.6 Binary Loader Resolution (Community/Enterprise) + +- ELF dynamic linking resolution +- PE import table analysis +- Mach-O load command parsing + +### 8.7 Feature Flag/Config Gating (Community/Enterprise) + +- Configuration-based path analysis +- Feature flag detection +- Conditional path evaluation + +### 8.8 Runtime Signal Correlation (Enterprise) + +- Zastava integration for runtime signals +- eBPF-based function tracing +- Actual execution path verification + +### 8.9 Gate Detection (Enterprise) + +- Authentication gate detection +- Authorization check identification +- Admin-only path detection + +### 8.10 Path Witness Generation (Enterprise) + +- Audit evidence for reachability claims +- Detailed path documentation +- Witness verification + +### 8.11 Reachability Drift Detection + +- Cross-version reachability comparison +- Path change detection +- Risk delta calculation + +### 8.12 Reachability Mini-Map API (Enterprise) + +- UI visualization data +- Compact graph representation +- Interactive exploration support + +### 8.13 Runtime Timeline API (Enterprise) + +- Temporal execution analysis +- Time-based function tracking +- Historical runtime data + +--- + +## 9. Binary Analysis + +### 9.1 Binary Identity Extraction + +- Build-ID extraction +- SHA-256 hash computation +- Content-addressed identification +- Metadata extraction + +### 9.2 Binary Format Parsers (Community/Enterprise) + +- **ELF** (Linux) parser +- **PE** (Windows) parser +- **Mach-O** (macOS) parser + +### 9.3 Build-ID Vulnerability Lookup + +- Direct build-ID to CVE mapping +- Pre-computed vulnerability databases + +### 9.4 Binary Corpus Support + +- **Debian/Ubuntu Corpus** (all tiers) +- **RPM/RHEL Corpus** (Community/Enterprise) + +### 9.5 Patch-Aware Backport Detection (Community/Enterprise) + +- Distribution patch tracking +- Backported fix detection +- False positive reduction + +### 9.6 Binary Fingerprint Generation (Enterprise) + +- Function-level fingerprints +- Code similarity hashing +- Version-independent matching + +### 9.7 Fingerprint Matching Engine (Enterprise) + +- Similarity search across binaries +- Fuzzy matching for modified code +- Large-scale fingerprint database + +### 9.8 DWARF/Symbol Analysis (Enterprise) + +- Debug symbol parsing +- Source location mapping +- Type information extraction + +### 9.9 Symbol Resolution (Symbols Module) + +- Symbol table parsing +- Name demangling +- Cross-reference building +- Symbol repository + +--- + +## 10. Policy Engine + +### 10.1 Policy Rule Formats + +- **YAML Policy Rules** (all tiers) +- **OPA/Rego Integration** (Enterprise) +- **Score Policy YAML** (Enterprise) + +### 10.2 Belnap K4 Four-Valued Logic + +- Unknown (no information) +- True (positive assertion) +- False (negative assertion) +- Conflict (contradictory assertions) + +### 10.3 Security Atoms (6 Types) + +- **PRESENT**: Package is present in artifact +- **APPLIES**: CVE applies to package version +- **REACHABLE**: Vulnerable code is reachable +- **MITIGATED**: Compensating controls exist +- **FIXED**: Vulnerability is fixed +- **MISATTRIBUTED**: CVE incorrectly assigned + +### 10.4 Policy Gates + +- **Minimum Confidence Gate**: Enforce minimum confidence threshold +- **Unknowns Budget Gate** (Community/Enterprise): Limit acceptable unknowns +- **Source Quota Gate** (Enterprise): 60% source cap enforcement +- **Reachability Requirement Gate** (Enterprise): Require reachability proof for criticals +- **Evidence Freshness Gate**: Enforce evidence age limits +- **VEX Trust Gate**: VEX-based policy decisions +- **Drift Gate**: Reachability drift enforcement +- **Stability Damping Gate**: Noise reduction + +### 10.5 Disposition Selection + +- ECMA-424 compliant disposition mapping +- Deterministic outcome selection +- Traceable decision paths + +### 10.6 Exception Objects & Workflow (Enterprise) + +- Time-bound exceptions +- Approval chain management +- Exception tracking + +### 10.7 Policy Version History (Enterprise) + +- Full policy change audit trail +- Policy rollback capability +- Version comparison + +### 10.8 Configurable Scoring Profiles (Enterprise) + +- Simple profile (basic scoring) +- Advanced profile (multi-factor scoring) +- Custom profile creation + +--- + +## 11. Attestation & Signing + +### 11.1 DSSE Envelope Signing + +- Detached signature envelopes +- Canonical JSON payloads +- Multi-signature support + +### 11.2 in-toto Statement Structure + +- Statement v1 format +- Subject binding to artifacts +- Predicate flexibility + +### 11.3 Attestation Predicates + +- **SBOM Predicate**: SBOM content attestation +- **VEX Predicate**: VEX statement attestation +- **Reachability Predicate** (Community/Enterprise): Reachability proof attestation +- **Policy Decision Predicate** (Community/Enterprise): Policy outcome attestation +- **Human Approval Predicate** (Enterprise): Manual approval attestation +- **Boundary Predicate** (Enterprise): Network exposure attestation + +### 11.4 Verdict Manifest + +- Signed verdict bundles (Community/Enterprise) +- Complete decision documentation +- Replay verification support + +### 11.5 Key Management + +- Ephemeral OIDC/keyless signing +- Short-lived key support +- HSM/KMS integration +- Key rotation management (Enterprise) + +### 11.6 SLSA Provenance (Enterprise) + +- SLSA v1.0 provenance attestations +- Build provenance capture +- Supply chain attestation + +### 11.7 Transparency Logging + +- **Rekor Transparency Log** (Enterprise): Public attestation logging +- **Cosign Integration** (Enterprise): Sigstore ecosystem compatibility +- Inclusion proof storage +- Local transparency mirror for offline + +--- + +## 12. Regional Cryptography + +### 12.1 Default Cryptography + +- **Ed25519** signing (default) +- Modern elliptic curve cryptography +- High performance signing/verification + +### 12.2 FIPS 140-2/3 Mode + +- ECDSA P-256 signing +- RSA-PSS signing +- US Federal compliance +- FIPS-validated modules + +### 12.3 eIDAS Signatures + +- ETSI TS 119 312 compliance +- EU qualified electronic signatures +- European compliance + +### 12.4 GOST/CryptoPro + +- GOST R 34.10-2012 signing +- Russian Federation compliance +- CryptoPro integration + +### 12.5 SM National Standard + +- GM/T 0003.2-2012 compliance +- SM2 signing algorithm +- China compliance + +### 12.6 Post-Quantum Cryptography + +- **Dilithium** signing (NIST PQC) +- **Falcon** signing support +- Future-proof security + +### 12.7 Crypto Plugin Architecture + +- Custom HSM integration +- Pluggable crypto providers +- Multi-signature DSSE envelopes (sign with multiple profiles) + +### 12.8 RootPack Bundles + +- Pre-configured trust root packages +- Regional trust root distribution +- Offline trust root updates + +--- + +## 13. Risk Scoring & Assessment + +### 13.1 Score Display + +- CVSS v4.0/v3.1/v2.0 display +- EPSS v4 probability display +- Composite risk scores + +### 13.2 Priority Band Classification + +- Critical/High/Medium/Low/Informational bands +- Configurable band thresholds +- Multi-factor classification + +### 13.3 EPSS-at-Scan Immutability (Community/Enterprise) + +- EPSS score captured at scan time +- Historical score preservation +- Score drift tracking + +### 13.4 Unified Confidence Model (Community/Enterprise) + +- 5-factor confidence scoring +- Source confidence weighting +- Evidence strength assessment + +### 13.5 Entropy-Based Scoring (Enterprise) + +- Information-theoretic risk assessment +- Uncertainty quantification + +### 13.6 Gate Multipliers (Enterprise) + +- Reachability-aware score adjustment +- Gate-based risk modification + +### 13.7 Unknowns Pressure Factor (Enterprise) + +- Uncertainty budget enforcement +- Unknown count impact on risk + +### 13.8 Custom Scoring Profiles (Enterprise) + +- Organization-specific scoring +- Factor weight customization +- Profile versioning + +### 13.9 Score Explanation Arrays + +- Per-finding score breakdown +- Factor contribution transparency +- Decision audit support + +--- + +## 14. Evidence Management + +### 14.1 Findings List + +- Comprehensive finding catalog +- Filtering and sorting +- Export capabilities + +### 14.2 Evidence Graph View + +- Visual evidence relationships +- Interactive exploration +- Dependency visualization + +### 14.3 Findings Ledger (Enterprise) + +- Immutable finding history +- Audit trail for all findings +- Finding lifecycle tracking + +### 14.4 Evidence Locker (Enterprise) + +- Sealed evidence storage +- Tamper-evident packaging +- Import/export capabilities + +### 14.5 Evidence TTL Policies (Enterprise) + +- Configurable retention rules +- Automatic expiration +- Compliance-driven retention + +### 14.6 Evidence Size Budgets (Enterprise) + +- Storage governance +- Quota enforcement +- Capacity planning + +### 14.7 Retention Tiers (Enterprise) + +- Hot tier (immediate access) +- Warm tier (near-line storage) +- Cold tier (archive storage) + +### 14.8 Privacy Controls (Enterprise) + +- Sensitive data redaction +- PII handling +- Anonymization support + +### 14.9 Audit Pack Export (Enterprise) + +- Compliance bundle generation +- Regulatory export formats +- Complete evidence packaging + +--- + +## 15. Determinism & Reproducibility + +### 15.1 Canonical JSON Serialization + +- RFC 8785 compliant serialization +- Sorted keys +- Minimal escaping +- Consistent number formatting + +### 15.2 Content-Addressed IDs + +- SHA-256 based identification +- Immutable references +- Deduplication support + +### 15.3 Replay Manifest (SRM) + +- Complete scan input capture +- Version pinning +- Configuration recording + +### 15.4 Replay Verification + +- `stella replay` CLI command +- Digest assertion +- Bit-for-bit comparison + +### 15.5 Evidence Freshness Multipliers (Community/Enterprise) + +- Age-based confidence adjustment +- Decay algorithms +- Freshness enforcement + +### 15.6 Proof Coverage Metrics (Community/Enterprise) + +- Evidence completeness measurement +- Gap identification +- Coverage reporting + +### 15.7 Fidelity Metrics (Enterprise) + +- **BF** (Base Fidelity): Input quality +- **SF** (Scan Fidelity): Detection quality +- **PF** (Proof Fidelity): Evidence quality +- Audit dashboard integration + +### 15.8 FN-Drift Rate Tracking (Enterprise) + +- False negative monitoring +- Quality trend analysis +- Alert thresholds + +### 15.9 Determinism Gate CI (Enterprise) + +- Automated determinism testing +- CI/CD integration +- Drift prevention + +--- + +## 16. CLI Features + +### 16.1 Core Commands + +- `stella scan` - Container image scanning +- `stella sbom` - SBOM generation and inspection +- `stella vex` - VEX evaluation and generation +- `stella advisory` - Advisory management +- `stella policy` - Policy evaluation +- `stella replay` - Deterministic replay + +### 16.2 SBOM Commands + +- `stella sbom generate` - Generate SBOM from image +- `stella sbom inspect` - View SBOM contents +- `stella sbom diff` - Compare SBOMs +- `stella sbom validate` - Validate SBOM format +- `stella sbom convert` - Convert between formats + +### 16.3 VEX Commands + +- `stella vex evaluate` - Evaluate VEX statements +- `stella vex generate` - Generate VEX documents +- `stella vex import` - Import VEX from file +- `stella vex export` - Export VEX statements + +### 16.4 Attestation Commands + +- `stella attest sign` - Sign attestations +- `stella attest verify` (Community/Enterprise) - Verify attestations +- `stella attest export` - Export attestations + +### 16.5 Reachability Commands + +- `stella reachability analyze` - Run reachability analysis +- `stella graph show` - Display reachability graph +- `stella reachability export` - Export reachability data + +### 16.6 Risk Commands + +- `stella risk evaluate` - Calculate risk scores +- `stella risk report` - Generate risk reports + +### 16.7 Policy Commands + +- `stella policy evaluate` - Run policy evaluation +- `stella policy validate` - Validate policy files +- `stella policy export` - Export policy decisions + +### 16.8 Offline Commands + +- `stella rootpack import` - Import trust root bundles +- `stella offline sync` - Sync offline data +- `stella offline verify` - Verify offline package + +### 16.9 Database Commands + +- `stella db update` - Update vulnerability database +- `stella db status` - Check database status +- `stella db export` - Export database snapshot + +### 16.10 Export Commands + +- `stella export sarif` - Export SARIF format +- `stella export json` - Export JSON format +- `stella export csv` - Export CSV format +- `stella export audit-pack` (Enterprise) - Export audit bundle + +### 16.11 Administrative Commands (Enterprise) + +- `stella admin` - Administrative utilities +- `stella symbols` - Symbol resolution commands +- `stella notify` - Notification management +- `stella orchestrator` - Workflow control + +### 16.12 CLI Technical Features + +- Native AOT compilation +- Cross-platform support (linux-x64, linux-arm64, osx-x64, osx-arm64, win-x64) +- Machine-readable output (JSON, NDJSON) +- Exit codes for CI/CD integration +- Environment variable configuration + +--- + +## 17. Web UI Features + +### 17.1 Core Interface + +- Dark/Light mode toggle +- Responsive design +- Locale support (Cyrillic, etc.) (Community/Enterprise) +- Keyboard shortcuts (Enterprise) + +### 17.2 Findings View + +- Findings Row Component +- Filtering and sorting +- Bulk actions +- Export capabilities + +### 17.3 Evidence Visualization + +- Evidence Drawer panel +- Proof Tab for attestations +- Evidence Graph View +- Confidence Meter + +### 17.4 VEX Interface + +- VEX Conflict Studio UI +- Claim Comparison Table (Enterprise) +- Trust Algebra Panel (Enterprise) + +### 17.5 Reachability Visualization + +- Reachability Mini-Map (Enterprise) +- Path visualization +- Call graph explorer + +### 17.6 Policy Interface + +- Policy Chips Display (Enterprise) +- Gate status visualization +- Policy decision trace + +### 17.7 Triage Features + +- Triage Canvas component +- Vulnerability triage workflow +- Status management +- Assignment capabilities + +### 17.8 Timeline Features (Enterprise) + +- Runtime Timeline view +- Historical execution data +- Temporal analysis + +### 17.9 Administrative Features (Enterprise) + +- Audit Trail UI +- Knowledge Snapshot UI (air-gap prep) +- Operator/Auditor Toggle (role separation) +- Reproduce Verdict Button + +### 17.10 Noise Gating UI + +- Delta visualization +- Gating statistics +- Noise reduction controls + +--- + +## 18. Offline & Air-Gap Operations + +### 18.1 Offline Update Kits (OUK) + +- Complete feed bundles +- Monthly (Community) / Weekly (Enterprise) updates +- Signed packages + +### 18.2 Knowledge Snapshots (Enterprise) + +- Sealed feed exports +- Complete knowledge state capture +- Merkle root verification + +### 18.3 Offline Signature Verification (Community/Enterprise) + +- Local verification without network +- Embedded revocation lists +- Cached trust roots + +### 18.4 Offline JWT Tokens (Enterprise) + +- 90-day offline tokens +- Local token validation +- Extended offline operation + +### 18.5 Air-Gap Bundle Manifest (Enterprise) + +- Transfer package specification +- Integrity verification +- Import/export workflows + +### 18.6 No-Egress Enforcement (Enterprise) + +- Strict network isolation +- Egress policy enforcement +- Connectivity validation + +### 18.7 Offline Components + +- Mirrored vulnerability feeds +- Local transparency log mirror +- RootPack trust bundles +- Embedded revocation lists + +### 18.8 One-Command Replay (Community/Enterprise) + +- `stella replay srm.yaml` for offline verification +- No network required for replay +- Complete evidence bundle + +--- + +## 19. Deployment Options + +### 19.1 Docker Compose + +- Single-node deployment (all tiers) +- Development environment setup +- Quick start configuration + +### 19.2 Helm Chart (Community/Enterprise) + +- Kubernetes deployment +- Configurable replicas +- Resource management +- Secret management + +### 19.3 High Availability (Enterprise) + +- Multi-replica deployment +- Load balancing +- Failover support +- Disaster recovery + +### 19.4 Horizontal Scaling (Enterprise) + +- Auto-scaling support +- Workload distribution +- Resource optimization + +### 19.5 Dedicated Capacity (Enterprise) + +- Reserved resources +- Guaranteed performance +- Isolation options + +### 19.6 Infrastructure Requirements + +- **PostgreSQL 16+**: Primary database +- **Valkey 8.0+**: Caching and queuing +- **RustFS (S3)** (Community/Enterprise): Object storage + +### 19.7 Container Images + +- Multi-architecture support (amd64, arm64) +- Minimal base images +- Regular security updates + +--- + +## 20. Authentication & Authorization + +### 20.1 Authentication Methods + +- **Basic Auth**: Username/password (all tiers) +- **API Keys**: Token-based access (all tiers) +- **SSO/SAML**: Okta, Azure AD integration (all tiers) +- **OIDC Support**: OpenID Connect with discovery (all tiers) + +### 20.2 OAuth 2.0 Grant Types + +- **Client Credentials**: Service-to-service authentication +- **Resource Owner Password Credentials**: User login +- **Authorization Code + PKCE**: Browser-based UI flows +- **Device Code**: CLI login on headless agents +- **Refresh Token Grant**: DPoP-bound or mTLS constrained + +### 20.3 Sender-Constraint Technologies + +#### DPoP (Demonstration of Proof-of-Possession) +- Proof JWT on every HTTP request +- Token bound via `cnf.jkt` (JWK thumbprint) +- Replay prevention with JTI cache +- Nonce support for high-value services + +#### mTLS (Mutual TLS Binding) +- Client certificate-bound tokens +- Token carries `cnf.x5t#S256` (cert thumbprint) +- Enforced for high-value audiences (Signer, Attestor) +- Certificate chain validation + +### 20.4 Token Management + +- **Access Token (OpTok)**: 120-300 second TTL +- **Refresh Tokens**: Optional, short-lived (≤ 8h), rotating +- Token refresh (12h Free / 30d Community / Annual Enterprise) +- Short-lived key support +- JWT format with custom claims + +### 20.5 Identity Provider Plugins + +- **Standard Plugin**: Local username/password, MFA support +- **LDAP Plugin**: Active Directory / OpenLDAP integration +- **OIDC Plugin**: External OIDC provider federation +- **SAML Plugin**: SAML 2.0 assertion processing + +### 20.6 RBAC (Role-Based Access Control) + +- **Basic RBAC**: User/Admin roles (all tiers) +- **Advanced RBAC** (Enterprise): Team-based scopes, custom roles +- 70+ granular permission scopes +- Scope-based authorization enforcement + +### 20.7 Scope Categories + +- **Authority Admin**: `authority:tenants.*`, `authority:users.*`, `authority:roles.*` +- **Scanner**: `scanner:read`, `scanner:scan`, `scanner:export` +- **Signer**: `signer:read`, `signer:sign`, `signer:rotate` +- **Policy**: `policy:write`, `policy:review`, `policy:approve`, `policy:publish` +- **VulnExplorer**: `vuln:view`, `vuln:investigate`, `vuln:operate` +- **VEX**: `vex:read`, `vex:ingest` +- **Graph**: `graph:read`, `graph:write`, `graph:export` +- **Evidence**: `evidence:create`, `evidence:read`, `evidence:hold` +- **Attestation**: `attest:read`, `attest:create`, `attest:admin` +- **Observability**: `obs:read`, `obs:incident`, `timeline:read` + +### 20.8 ABAC (Attribute-Based Access Control) + +- Environment attribute filtering (`stellaops:attr:env`) +- Ownership visibility (`stellaops:attr:owner`) +- Business tier filtering (`stellaops:attr:business_tier`) + +### 20.9 Multi-Tenant Management (Enterprise) + +- Organization hierarchy +- Tenant isolation via `tid` claim +- Installation isolation via `inst` claim +- Cross-tenant policy enforcement + +### 20.10 Specialized Tokens + +- **Incident Mode Tokens**: 5-minute freshness, requires human reason +- **Vulnerability Workflow Tokens**: Anti-forgery for mutations +- **Attachment Access Tokens**: Evidence bundle downloads +- **Acknowledgment Tokens**: Notification workflows + +### 20.11 Security Features + +- Password lockout with configurable attempts +- Key rotation (30-90 day cadence, zero-downtime) +- KMS/HSM support (private keys never leave) +- Rate limiting (per-client, per-IP, per-endpoint) +- PKCE required for Authorization Code flow + +### 20.12 Audit Logging (Enterprise) + +- Token issuance audit (sub, aud, scopes, tid, jti) +- Revocation events +- Admin changes (client/user/role) +- Credential attempt tracking with failure codes +- DPoP/mTLS validation events +- SIEM integration +- User activity tracking + +--- + +## 21. Integrations & Notifications + +### 21.1 Notification Channels + +- **In-App Notifications** (all tiers) +- **Email Notifications** (Community/Enterprise) +- **Slack Integration** (all tiers) +- **Microsoft Teams Integration** (all tiers) + +### 21.2 Alert Types + +- New vulnerability alerts +- EPSS change alerts (Community/Enterprise) +- Policy violation alerts +- Scan completion notifications + +### 21.3 Registry Integration + +- **Zastava Registry Hooks**: Auto-scan on container push (all tiers) +- Registry webhook observer +- Event-driven scanning + +### 21.4 CI/CD Integration (Enterprise) + +- GitLab CI/CD gates +- GitHub Actions integration +- Jenkins plugin +- Custom webhook endpoints + +### 21.5 Custom Webhooks (Enterprise) + +- Configurable endpoints +- Event filtering +- Payload customization + +### 21.6 Enterprise Connectors (Enterprise) + +- Grid/Premium API access +- Custom connector development +- Third-party integration support + +### 21.7 Gateway & Router + +- API gateway with routing +- Transport abstraction (TCP/TLS/UDP/RabbitMQ/Valkey) +- Rate limiting +- Request routing + +--- + +## 22. Observability & Telemetry + +### 22.1 Metrics + +- Basic metrics (all tiers) +- Scan performance metrics +- Resource utilization metrics +- Error rate tracking + +### 22.2 OpenTelemetry (Enterprise) + +- Full distributed tracing +- Trace context propagation +- Custom span attributes + +### 22.3 Prometheus Export (Enterprise) + +- Prometheus metric format +- Custom metrics endpoints +- Grafana dashboard support + +### 22.4 Telemetry Options + +- Opt-in telemetry (all tiers) +- Telemetry configuration +- Privacy controls + +### 22.5 Quality KPIs Dashboard (Enterprise) + +- Triage metrics +- Detection accuracy +- Coverage statistics + +### 22.6 SLA Monitoring (Enterprise) + +- Uptime tracking +- Performance monitoring +- SLA compliance reporting + +### 22.7 Logging + +- Structured logging +- Log levels configuration +- Log aggregation support + +--- + +## 23. Scheduling & Automation + +### 23.1 Manual Scans + +- On-demand scanning (all tiers) +- CLI-triggered scans +- UI-initiated scans + +### 23.2 Scheduled Scans (Enterprise) + +- Cron-based scheduling +- Recurring scan configuration +- Schedule management + +### 23.3 Event-Driven Scanning (Enterprise) + +- Registry push triggers +- Webhook-initiated scans +- Pipeline integration + +### 23.4 Task Pack Orchestration (Enterprise) + +- Declarative workflow definition +- Task pack execution +- Plan-hash binding +- Approval gates +- Sealed mode for air-gap + +### 23.5 EPSS Daily Refresh (Enterprise) + +- Automatic EPSS updates +- Score recalculation +- Delta notifications + +### 23.6 Scheduler Features + +- Job queue management +- Priority scheduling +- Resource allocation +- Failure retry policies + +### 23.7 Orchestrator Features + +- Workflow coordination +- Task dependency management +- Parallel execution +- Status tracking + +--- + +## 24. Version Comparison + +### 24.1 Package Version Formats + +- **RPM (NEVRA)**: Name-Epoch-Version-Release-Architecture +- **Debian (EVR)**: Epoch-Version-Release +- **Alpine (APK)**: Alpine package versioning +- **SemVer**: Semantic versioning (major.minor.patch) + +### 24.2 PURL Resolution + +- Package URL parsing +- Ecosystem-aware resolution +- Version normalization + +### 24.3 Version Range Matching + +- Affected version range detection +- Fixed version identification +- Upgrade path calculation + +--- + +## 25. Database & Storage + +### 25.1 PostgreSQL Features + +- PostgreSQL 16+ support +- Per-module schema isolation +- Row-Level Security (RLS) for multi-tenancy +- Connection pooling + +### 25.2 Valkey/Redis Features + +- Valkey 8.0+ support +- Caching layer +- Job queue backend +- Session storage + +### 25.3 Object Storage (RustFS/S3) + +- S3-compatible storage (Community/Enterprise) +- Content-addressed blob storage +- SBOM/evidence storage +- Artifact storage + +### 25.4 Storage Features + +- Content deduplication +- Compression support +- Encryption at rest +- Retention policies + +--- + +## 26. API Capabilities + +### 26.1 REST API + +- RESTful endpoints +- OpenAPI 3.0 specification +- JSON request/response +- Pagination support + +### 26.2 API Features + +- Rate limiting (all tiers) +- 429 Backpressure handling +- Retry-After headers +- Priority queue (Enterprise) +- Burst allowance (Enterprise) + +### 26.3 Quota Management + +- Usage API (`/quota`) +- Scan quota tracking +- Quota enforcement +- Custom quotas (Enterprise) + +### 26.4 API Authentication + +- API key authentication +- JWT bearer tokens +- OAuth 2.0 support +- DPoP support + +--- + +## 27. Support & Services + +### 27.1 Documentation + +- Comprehensive documentation (all tiers) +- API reference +- Architecture guides +- Tutorials and guides + +### 27.2 Community Support + +- Community forums (all tiers) +- GitHub Issues (all tiers) +- Documentation wiki + +### 27.3 Email Support (Enterprise) + +- Business hours support +- Ticket-based support + +### 27.4 Priority Support (Enterprise) + +- 4-hour response time +- Priority ticket handling + +### 27.5 24/7 Critical Support (Enterprise) + +- Round-the-clock support (add-on) +- Emergency response + +### 27.6 Dedicated CSM (Enterprise) + +- Named customer success manager +- Regular check-ins +- Account management + +### 27.7 Professional Services (Enterprise) + +- Implementation assistance +- Custom development +- Architecture review + +### 27.8 Training & Certification (Enterprise) + +- Team enablement +- Certification programs +- Custom training + +### 27.9 SLA Guarantee (Enterprise) + +- 99.9% uptime guarantee +- SLA credits +- Performance guarantees + +--- + +## Appendix A: Module Reference + +| Module | Description | +|--------|-------------| +| **Authority** | Authentication, authorization, OAuth/OIDC, DPoP | +| **Gateway** | API gateway with routing and transport abstraction | +| **Router** | Transport-agnostic messaging | +| **Concelier** | Vulnerability advisory ingestion and merge engine | +| **Excititor** | VEX document ingestion and export | +| **VexLens** | VEX consensus computation across issuers | +| **VexHub** | VEX distribution and exchange hub | +| **IssuerDirectory** | Issuer trust registry | +| **Feedser** | Evidence collection for backport detection | +| **Mirror** | Vulnerability feed mirror and distribution | +| **Scanner** | Container scanning with SBOM generation | +| **BinaryIndex** | Binary identity extraction and fingerprinting | +| **AdvisoryAI** | AI-assisted advisory analysis | +| **ReachGraph** | Reachability graph service | +| **Symbols** | Symbol resolution and debug information | +| **Attestor** | in-toto/DSSE attestation generation | +| **Signer** | Cryptographic signing operations | +| **SbomService** | SBOM storage, versioning, and lineage ledger | +| **EvidenceLocker** | Sealed evidence storage and export | +| **ExportCenter** | Batch export and report generation | +| **Provenance** | SLSA/DSSE attestation tooling | +| **Policy** | Policy engine with K4 lattice logic | +| **RiskEngine** | Risk scoring runtime | +| **VulnExplorer** | Vulnerability exploration and triage UI backend | +| **Unknowns** | Unknown component and symbol tracking | +| **Scheduler** | Job scheduling and queue management | +| **Orchestrator** | Workflow orchestration and task coordination | +| **TaskRunner** | Task pack execution engine | +| **Notify** | Notification toolkit | +| **Notifier** | Notifications Studio host | +| **PacksRegistry** | Task packs registry and distribution | +| **TimelineIndexer** | Timeline event indexing | +| **Replay** | Deterministic replay engine | +| **CLI** | Command-line interface | +| **Zastava** | Container registry webhook observer | +| **Web** | Angular frontend SPA | +| **Cryptography** | Crypto plugins (FIPS, eIDAS, GOST, SM, PQ) | +| **Telemetry** | OpenTelemetry traces, metrics, logging | +| **Graph** | Call graph and reachability data structures | +| **Signals** | Runtime signal collection and correlation | +| **AirGap** | Air-gapped deployment support | +| **AOC** | Append-Only Contract enforcement | + +--- + +## Appendix B: Supported Standards + +| Standard | Version | Usage | +|----------|---------|-------| +| CycloneDX | 1.7 | Primary SBOM format | +| SPDX | 3.0.1 | SBOM format | +| in-toto | Statement v1 | Attestation format | +| DSSE | v1 | Envelope signing | +| OpenVEX | Current spec | VEX format | +| SARIF | 2.1.0 | Findings interchange | +| Sigstore Rekor | API stable | Transparency logging | +| SLSA | v1.0 | Provenance attestation | + +--- + +## Appendix C: Glossary + +| Term | Definition | +|------|------------| +| **SBOM** | Software Bill of Materials - component inventory | +| **VEX** | Vulnerability Exploitability eXchange - exploitability status | +| **DSSE** | Dead Simple Signing Envelope - detached signatures | +| **in-toto** | Software supply chain attestation framework | +| **K4 Lattice** | Belnap four-valued logic (Unknown, True, False, Conflict) | +| **SRM** | Scan Replay Manifest - deterministic replay bundle | +| **PURL** | Package URL - universal package identifier | +| **NEVRA** | Name-Epoch-Version-Release-Architecture (RPM) | +| **EVR** | Epoch-Version-Release (Debian) | +| **KEV** | Known Exploited Vulnerabilities | +| **EPSS** | Exploit Prediction Scoring System | +| **OVAL** | Open Vulnerability and Assessment Language | + +--- + +*Last updated: 4 Jan 2026* +*For tier availability, see [`04_FEATURE_MATRIX.md`](04_FEATURE_MATRIX.md)* diff --git a/docs/implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md b/docs/implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md new file mode 100644 index 000000000..683734446 --- /dev/null +++ b/docs/implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md @@ -0,0 +1,540 @@ +# Sprint 20260104_002_SCANNER - Secret Leak Detection Core Analyzer + +## Topic & Scope + +Implement the core `StellaOps.Scanner.Analyzers.Secrets` plugin that detects accidentally committed secrets in container layers during scans. This is the foundational sprint for secret leak detection capability. + +**Key deliverables:** +1. **Secrets Analyzer Plugin**: Core analyzer that executes regex/entropy-based detection rules +2. **Rule Engine**: Rule definition models, matching logic, and deterministic execution +3. **Masking Engine**: Payload masking to ensure secrets never leak in outputs +4. **Evidence Emission**: `secret.leak` evidence type integration with ScanAnalysisStore +5. **Feature Flag**: Experimental toggle for gradual rollout + +**Working directory:** `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/` + +## Dependencies & Concurrency + +- **Depends on**: Surface.Secrets, Surface.Validation, Surface.Env (already implemented) +- **Required by**: Sprint 20260104_003 (Rule Bundle Infrastructure), Sprint 20260104_004 (Policy DSL) +- **Parallel work**: Tasks SLD-001 through SLD-008 can be developed concurrently +- **Integration tasks** (SLD-009+) require prior tasks complete + +## Documentation Prerequisites + +- docs/README.md +- docs/07_HIGH_LEVEL_ARCHITECTURE.md +- docs/modules/scanner/architecture.md +- docs/modules/scanner/design/surface-secrets.md +- docs/modules/scanner/operations/secret-leak-detection.md (target spec) +- CLAUDE.md (especially Section 8: Code Quality & Determinism Rules) + +## Delivery Tracker + +| # | Task ID | Status | Key dependency / next step | Owners | Task Definition | +| --- | --- | --- | --- | --- | --- | +| 1 | SLD-001 | TODO | None | Scanner Guild | Create project structure and csproj | +| 2 | SLD-002 | TODO | None | Scanner Guild | Define SecretRule and SecretRuleset models | +| 3 | SLD-003 | TODO | None | Scanner Guild | Implement ISecretDetector interface and RegexDetector | +| 4 | SLD-004 | TODO | None | Scanner Guild | Implement EntropyDetector for high-entropy string detection | +| 5 | SLD-005 | TODO | None | Scanner Guild | Implement PayloadMasker with configurable masking strategies | +| 6 | SLD-006 | TODO | None | Scanner Guild | Define SecretLeakEvidence record and finding model | +| 7 | SLD-007 | TODO | SLD-002 | Scanner Guild | Implement RulesetLoader with JSON parsing | +| 8 | SLD-008 | TODO | None | Scanner Guild | Add SecretsAnalyzerOptions with feature flag support | +| 9 | SLD-009 | TODO | SLD-003,SLD-004 | Scanner Guild | Implement CompositeSecretDetector combining regex and entropy | +| 10 | SLD-010 | TODO | SLD-006,SLD-009 | Scanner Guild | Implement SecretsAnalyzer (ILanguageAnalyzer) | +| 11 | SLD-011 | TODO | SLD-010 | Scanner Guild | Add SecretsAnalyzerHost for plugin lifecycle | +| 12 | SLD-012 | TODO | SLD-011 | Scanner Guild | Integrate with Scanner Worker pipeline | +| 13 | SLD-013 | TODO | SLD-010 | Scanner Guild | Add DI registration in ServiceCollectionExtensions | +| 14 | SLD-014 | TODO | All | Scanner Guild | Add comprehensive unit tests | +| 15 | SLD-015 | TODO | SLD-014 | Scanner Guild | Add integration tests with test fixtures | +| 16 | SLD-016 | TODO | All | Scanner Guild | Create AGENTS.md for module | + +## Task Details + +### SLD-001: Project Structure + +Create the project skeleton following Scanner conventions: + +``` +src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/ +├── StellaOps.Scanner.Analyzers.Secrets.csproj +├── AGENTS.md +├── AssemblyInfo.cs +├── Detectors/ +│ ├── ISecretDetector.cs +│ ├── RegexDetector.cs +│ ├── EntropyDetector.cs +│ └── CompositeSecretDetector.cs +├── Rules/ +│ ├── SecretRule.cs +│ ├── SecretRuleset.cs +│ └── RulesetLoader.cs +├── Masking/ +│ ├── IPayloadMasker.cs +│ └── PayloadMasker.cs +├── Evidence/ +│ ├── SecretLeakEvidence.cs +│ └── SecretFinding.cs +├── SecretsAnalyzer.cs +├── SecretsAnalyzerHost.cs +├── SecretsAnalyzerOptions.cs +└── ServiceCollectionExtensions.cs +``` + +csproj should reference: +- StellaOps.Scanner.Core +- StellaOps.Scanner.Surface +- StellaOps.Evidence.Core + +### SLD-002: Rule Models + +Define the rule structure for secret detection: + +```csharp +/// +/// A single secret detection rule. +/// +public sealed record SecretRule +{ + public required string Id { get; init; } // e.g., "stellaops.secrets.aws-access-key" + public required string Version { get; init; } // e.g., "2025.11.0" + public required string Name { get; init; } // Human-readable name + public required string Description { get; init; } + public required SecretRuleType Type { get; init; } // Regex, Entropy, Composite + public required string Pattern { get; init; } // Regex pattern or entropy config + public required SecretSeverity Severity { get; init; } + public required SecretConfidence Confidence { get; init; } + public string? MaskingHint { get; init; } // e.g., "prefix:4,suffix:2" + public ImmutableArray Keywords { get; init; } // Pre-filter keywords + public ImmutableArray FilePatterns { get; init; } // Glob patterns for file filtering + public bool Enabled { get; init; } = true; +} + +public enum SecretRuleType { Regex, Entropy, Composite } +public enum SecretSeverity { Low, Medium, High, Critical } +public enum SecretConfidence { Low, Medium, High } + +/// +/// A versioned collection of secret detection rules. +/// +public sealed record SecretRuleset +{ + public required string Id { get; init; } // e.g., "secrets.ruleset" + public required string Version { get; init; } // e.g., "2025.11" + public required DateTimeOffset CreatedAt { get; init; } + public required ImmutableArray Rules { get; init; } + public string? Sha256Digest { get; init; } // Integrity hash +} +``` + +Location: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/` + +### SLD-003: Regex Detector + +Implement regex-based secret detection: + +```csharp +public interface ISecretDetector +{ + string DetectorId { get; } + + ValueTask> DetectAsync( + ReadOnlyMemory content, + string filePath, + SecretRule rule, + CancellationToken ct = default); +} + +public sealed record SecretMatch( + SecretRule Rule, + string FilePath, + int LineNumber, + int ColumnStart, + int ColumnEnd, + ReadOnlyMemory RawMatch, // For masking + double ConfidenceScore); + +public sealed class RegexDetector : ISecretDetector +{ + public string DetectorId => "regex"; + + // Implementation notes: + // - Use compiled regex for performance + // - Apply keyword pre-filter before regex matching + // - Respect file pattern filters + // - Track line/column for precise location + // - Never log raw match content +} +``` + +Location: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/` + +### SLD-004: Entropy Detector + +Implement Shannon entropy-based detection for high-entropy strings: + +```csharp +public sealed class EntropyDetector : ISecretDetector +{ + public string DetectorId => "entropy"; + + // Implementation notes: + // - Calculate Shannon entropy for candidate strings + // - Default threshold: 4.5 bits per character + // - Minimum length: 16 characters + // - Skip common high-entropy non-secrets (UUIDs, hashes in comments) + // - Apply charset detection (base64, hex, alphanumeric) +} + +public static class EntropyCalculator +{ + /// + /// Calculates Shannon entropy in bits per character. + /// + public static double Calculate(ReadOnlySpan data) + { + // Use CultureInfo.InvariantCulture for all formatting + // Return 0.0 for empty input + } +} +``` + +### SLD-005: Payload Masker + +Implement secure payload masking: + +```csharp +public interface IPayloadMasker +{ + /// + /// Masks a secret payload preserving prefix/suffix for identification. + /// + /// The raw secret bytes + /// Optional masking hint from rule (e.g., "prefix:4,suffix:2") + /// Masked string (e.g., "AKIA****B7") + string Mask(ReadOnlySpan payload, string? hint = null); +} + +public sealed class PayloadMasker : IPayloadMasker +{ + // Default: preserve first 4 and last 2 characters + // Replace middle with asterisks (max 8 asterisks) + // Minimum output length: 8 characters + // Never expose more than 6 characters total + + public const int DefaultPrefixLength = 4; + public const int DefaultSuffixLength = 2; + public const int MaxMaskLength = 8; + public const char MaskChar = '*'; +} +``` + +### SLD-006: Evidence Models + +Define the evidence structure for policy integration: + +```csharp +/// +/// Evidence record for a detected secret leak. +/// +public sealed record SecretLeakEvidence +{ + public required string EvidenceType => "secret.leak"; + public required string RuleId { get; init; } + public required string RuleVersion { get; init; } + public required SecretSeverity Severity { get; init; } + public required SecretConfidence Confidence { get; init; } + public required string FilePath { get; init; } + public required int LineNumber { get; init; } + public required string Mask { get; init; } // Masked payload + public required string BundleId { get; init; } + public required string BundleVersion { get; init; } + public required DateTimeOffset DetectedAt { get; init; } + public ImmutableDictionary? Metadata { get; init; } +} + +/// +/// Aggregated finding for a single secret match. +/// +public sealed record SecretFinding +{ + public required Guid Id { get; init; } + public required SecretLeakEvidence Evidence { get; init; } + public required string ScanId { get; init; } + public required string TenantId { get; init; } + public required string ArtifactDigest { get; init; } +} +``` + +### SLD-007: Ruleset Loader + +Implement deterministic ruleset loading: + +```csharp +public interface IRulesetLoader +{ + ValueTask LoadAsync( + string rulesetPath, + CancellationToken ct = default); + + ValueTask LoadFromJsonlAsync( + Stream rulesStream, + string bundleId, + string bundleVersion, + CancellationToken ct = default); +} + +public sealed class RulesetLoader : IRulesetLoader +{ + // Implementation notes: + // - Parse secrets.ruleset.rules.jsonl (NDJSON format) + // - Validate rule schema on load + // - Sort rules by ID for deterministic ordering + // - Calculate and verify SHA-256 digest + // - Use CultureInfo.InvariantCulture for all parsing + // - Log bundle version on successful load +} +``` + +### SLD-008: Analyzer Options + +Configuration options with feature flag: + +```csharp +public sealed class SecretsAnalyzerOptions +{ + /// + /// Enable secret leak detection (experimental feature). + /// + public bool Enabled { get; set; } = false; + + /// + /// Path to the ruleset bundle directory. + /// + public string RulesetPath { get; set; } = "/opt/stellaops/plugins/scanner/analyzers/secrets"; + + /// + /// Minimum confidence level to report findings. + /// + public SecretConfidence MinConfidence { get; set; } = SecretConfidence.Medium; + + /// + /// Maximum findings per scan (circuit breaker). + /// + public int MaxFindingsPerScan { get; set; } = 1000; + + /// + /// File size limit for scanning (bytes). + /// + public long MaxFileSizeBytes { get; set; } = 10 * 1024 * 1024; // 10MB + + /// + /// Enable entropy-based detection. + /// + public bool EnableEntropyDetection { get; set; } = true; + + /// + /// Entropy threshold (bits per character). + /// + public double EntropyThreshold { get; set; } = 4.5; +} +``` + +### SLD-009: Composite Detector + +Combine multiple detection strategies: + +```csharp +public sealed class CompositeSecretDetector : ISecretDetector +{ + private readonly IReadOnlyList _detectors; + private readonly ILogger _logger; + + public string DetectorId => "composite"; + + // Implementation notes: + // - Execute detectors in parallel where possible + // - Deduplicate overlapping matches + // - Merge confidence scores for overlapping detections + // - Respect per-rule detector type preference +} +``` + +### SLD-010: Secrets Analyzer + +Main analyzer implementation: + +```csharp +public sealed class SecretsAnalyzer : ILayerAnalyzer +{ + public string AnalyzerId => "secrets"; + public string DisplayName => "Secret Leak Detector"; + + // Implementation notes: + // - Check feature flag before processing + // - Load ruleset once at startup (cached) + // - Apply file pattern filters efficiently + // - Execute detection on text files only + // - Emit SecretLeakEvidence for each finding + // - Apply masking before any output + // - Track metrics: scanner.secret.finding_total + // - Add tracing span: scanner.secrets.scan +} +``` + +### SLD-011: Analyzer Host + +Lifecycle management for the analyzer: + +```csharp +public sealed class SecretsAnalyzerHost : IHostedService +{ + // Implementation notes: + // - Load and validate ruleset on startup + // - Log bundle version and rule count + // - Verify DSSE signature if available + // - Graceful shutdown with finding flush + // - Emit startup log: "SecretsAnalyzerHost: Loaded bundle {version} with {count} rules" +} +``` + +### SLD-012: Worker Integration + +Integrate with Scanner Worker pipeline: + +```csharp +// In Scanner.Worker processing pipeline: +// 1. Add SecretsAnalyzer to analyzer chain (after language analyzers) +// 2. Gate execution on feature flag +// 3. Store findings in ScanAnalysisStore +// 4. Include in scan completion event +``` + +### SLD-013: DI Registration + +```csharp +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddSecretsAnalyzer( + this IServiceCollection services, + IConfiguration configuration) + { + services.AddOptions() + .Bind(configuration.GetSection("Scanner:Analyzers:Secrets")) + .ValidateDataAnnotations() + .ValidateOnStart(); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + + return services; + } +} +``` + +### SLD-014: Unit Tests + +Required test coverage in `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/`: + +``` +├── Detectors/ +│ ├── RegexDetectorTests.cs +│ ├── EntropyDetectorTests.cs +│ ├── EntropyCalculatorTests.cs +│ └── CompositeSecretDetectorTests.cs +├── Rules/ +│ ├── SecretRuleTests.cs +│ └── RulesetLoaderTests.cs +├── Masking/ +│ └── PayloadMaskerTests.cs +├── Evidence/ +│ └── SecretLeakEvidenceTests.cs +├── SecretsAnalyzerTests.cs +└── Fixtures/ + ├── aws-access-key.txt + ├── github-token.txt + ├── private-key.pem + └── test-ruleset.jsonl +``` + +Test requirements: +- All tests must be deterministic +- Use `[Trait("Category", "Unit")]` for unit tests +- Test masking never exposes full secrets +- Test entropy calculation with known inputs +- Test regex patterns match expected secrets + +### SLD-015: Integration Tests + +Integration tests with Scanner Worker: + +``` +├── SecretsAnalyzerIntegrationTests.cs +│ - Test full scan with secrets embedded +│ - Verify findings in ScanAnalysisStore +│ - Verify masking in output +│ - Test feature flag disables analyzer +├── RulesetLoadingTests.cs +│ - Test loading from file system +│ - Test invalid ruleset handling +│ - Test missing bundle handling +``` + +### SLD-016: Module AGENTS.md + +Create `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AGENTS.md` with: +- Mission statement +- Scope definition +- Required reading list +- Working agreements +- Security considerations + +## Built-in Rule Examples + +Initial rules to include in default bundle: + +| Rule ID | Pattern Type | Description | +|---------|--------------|-------------| +| `stellaops.secrets.aws-access-key` | Regex | AWS Access Key ID (AKIA...) | +| `stellaops.secrets.aws-secret-key` | Regex + Entropy | AWS Secret Access Key | +| `stellaops.secrets.github-pat` | Regex | GitHub Personal Access Token | +| `stellaops.secrets.github-app` | Regex | GitHub App Token (ghs_, ghp_) | +| `stellaops.secrets.gitlab-pat` | Regex | GitLab Personal Access Token | +| `stellaops.secrets.private-key-rsa` | Regex | RSA Private Key (PEM) | +| `stellaops.secrets.private-key-ec` | Regex | EC Private Key (PEM) | +| `stellaops.secrets.jwt` | Regex + Entropy | JSON Web Token | +| `stellaops.secrets.basic-auth` | Regex | Basic Auth credentials in URLs | +| `stellaops.secrets.generic-api-key` | Entropy | High-entropy API key patterns | + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Use NDJSON for rule format | Line-based parsing, easy streaming, git-friendly diffs | +| Mask before any persistence | Defense in depth - secrets never stored | +| Feature flag default off | Safe rollout, tenant opt-in required | +| Entropy threshold 4.5 bits | Balance between false positives and detection rate | +| Max 1000 findings per scan | Circuit breaker prevents DoS on noisy images | +| Text files only | Binary secret detection deferred to future sprint | + +## Metrics & Observability + +| Metric | Type | Labels | +|--------|------|--------| +| `scanner.secret.finding_total` | Counter | tenant, ruleId, severity, confidence | +| `scanner.secret.scan_duration_seconds` | Histogram | tenant | +| `scanner.secret.rules_loaded` | Gauge | bundleVersion | +| `scanner.secret.files_scanned` | Counter | tenant | + +## Execution Log + +| Date | Action | Notes | +|------|--------|-------| +| 2026-01-04 | Sprint created | Based on gap analysis of secrets scanning support | + diff --git a/docs/implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md b/docs/implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md new file mode 100644 index 000000000..b61d59dce --- /dev/null +++ b/docs/implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md @@ -0,0 +1,451 @@ +# Sprint 20260104_003_SCANNER - Secret Detection Rule Bundle Infrastructure + +## Topic & Scope + +Implement the DSSE-signed rule bundle infrastructure for secret leak detection. This sprint delivers the signing, verification, and distribution pipeline for deterministic rule bundles. + +**Key deliverables:** +1. **Bundle Schema**: Formal JSON schema for rule bundles +2. **Bundle Builder**: CLI tool to create and sign rule bundles +3. **DSSE Integration**: Signing and verification using existing Signer/Attestor modules +4. **Bundle Verification**: Runtime verification of bundle integrity and provenance +5. **Default Bundle**: Initial set of secret detection rules + +**Working directory:** `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/`, `src/Cli/`, `offline/rules/secrets/` + +## Dependencies & Concurrency + +- **Depends on**: Sprint 20260104_002 (Core Analyzer), StellaOps.Attestor, StellaOps.Signer +- **Required by**: Sprint 20260104_005 (Offline Kit Integration) +- **Parallel work**: Tasks RB-001 through RB-005 can be developed concurrently + +## Documentation Prerequisites + +- docs/modules/scanner/operations/secret-leak-detection.md +- docs/modules/attestor/architecture.md +- docs/modules/signer/architecture.md +- docs/ci/dsse-build-flow.md +- CLAUDE.md Section 8.6 (DSSE PAE Consistency) + +## Delivery Tracker + +| # | Task ID | Status | Key dependency / next step | Owners | Task Definition | +| --- | --- | --- | --- | --- | --- | +| 1 | RB-001 | DONE | None | Scanner Guild | Define bundle manifest JSON schema | +| 2 | RB-002 | DONE | None | Scanner Guild | Define rules JSONL schema and validation | +| 3 | RB-003 | DONE | RB-001,RB-002 | Scanner Guild | Create BundleBuilder class for bundle creation | +| 4 | RB-004 | DONE | RB-003 | Scanner Guild | Add DSSE signing integration | +| 5 | RB-005 | DONE | RB-004 | Scanner Guild | Implement BundleVerifier with Attestor integration | +| 6 | RB-006 | DONE | RB-005 | CLI Guild | Add `stella secrets bundle create` CLI command | +| 7 | RB-007 | DONE | RB-005 | CLI Guild | Add `stella secrets bundle verify` CLI command | +| 8 | RB-008 | DONE | RB-005 | Scanner Guild | Integrate verification into SecretsAnalyzerHost | +| 9 | RB-009 | DONE | RB-002 | Scanner Guild | Create default rule definitions | +| 10 | RB-010 | DONE | RB-009 | Scanner Guild | Build and sign initial bundle (2026.01) | +| 11 | RB-011 | DONE | All | Scanner Guild | Add unit and integration tests | +| 12 | RB-012 | DONE | RB-010 | Docs Guild | Document bundle lifecycle and rotation | + +## Task Details + +### RB-001: Bundle Manifest Schema + +Define the manifest schema (`secrets.ruleset.manifest.json`): + +```json +{ + "$schema": "https://stellaops.io/schemas/secrets-ruleset-manifest-v1.json", + "schemaVersion": "1.0", + "id": "secrets.ruleset", + "version": "2026.01", + "createdAt": "2026-01-04T00:00:00Z", + "description": "StellaOps Secret Detection Rules", + "rules": [ + { + "id": "stellaops.secrets.aws-access-key", + "version": "1.0.0", + "severity": "high", + "enabled": true + } + ], + "integrity": { + "rulesFile": "secrets.ruleset.rules.jsonl", + "rulesSha256": "abc123...", + "totalRules": 15, + "enabledRules": 15 + }, + "signatures": { + "dsseEnvelope": "secrets.ruleset.dsse.json" + } +} +``` + +Location: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/Schemas/` + +### RB-002: Rules JSONL Schema + +Define the rule entry schema for NDJSON format: + +```json +{ + "id": "stellaops.secrets.aws-access-key", + "version": "1.0.0", + "name": "AWS Access Key ID", + "description": "Detects AWS Access Key IDs in source code and configuration files", + "type": "regex", + "pattern": "(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}", + "severity": "high", + "confidence": "high", + "keywords": ["AKIA", "ASIA", "AIDA", "aws"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.conf", "*.config"], + "maskingHint": "prefix:4,suffix:2", + "metadata": { + "category": "cloud-credentials", + "provider": "aws", + "references": ["https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html"] + } +} +``` + +Validation rules: +- `id` must be namespaced (e.g., `stellaops.secrets.*`) +- `version` must be valid SemVer +- `pattern` must be valid regex (compile-time validation) +- `severity` must be one of: low, medium, high, critical +- `confidence` must be one of: low, medium, high + +### RB-003: Bundle Builder + +Implement the bundle creation logic: + +```csharp +public interface IBundleBuilder +{ + /// + /// Creates a bundle from individual rule files. + /// + Task BuildAsync( + BundleBuildOptions options, + CancellationToken ct = default); +} + +public sealed record BundleBuildOptions +{ + public required string OutputDirectory { get; init; } + public required string BundleId { get; init; } + public required string Version { get; init; } + public required IReadOnlyList RuleFiles { get; init; } + public TimeProvider? TimeProvider { get; init; } +} + +public sealed record BundleArtifact +{ + public required string ManifestPath { get; init; } + public required string RulesPath { get; init; } + public required string RulesSha256 { get; init; } + public required int TotalRules { get; init; } +} + +public sealed class BundleBuilder : IBundleBuilder +{ + // Implementation notes: + // - Validate each rule on load + // - Sort rules by ID for deterministic output + // - Compute SHA-256 of rules file + // - Generate manifest with integrity info + // - Use TimeProvider for timestamps (determinism) +} +``` + +Location: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/` + +### RB-004: DSSE Signing Integration + +Integrate with Signer module for bundle signing: + +```csharp +public interface IBundleSigner +{ + /// + /// Signs a bundle artifact producing a DSSE envelope. + /// + Task SignAsync( + BundleArtifact artifact, + SigningOptions options, + CancellationToken ct = default); +} + +public sealed record SigningOptions +{ + public required string KeyId { get; init; } + public string PayloadType { get; init; } = "application/vnd.stellaops.secrets-ruleset+json"; + public bool IncludeCertificateChain { get; init; } = true; +} + +public sealed class BundleSigner : IBundleSigner +{ + private readonly ISigner _signer; + + // Implementation notes: + // - Use existing Signer infrastructure + // - Payload is the manifest JSON (not rules file) + // - Include rules file digest in signed payload + // - Support multiple signature algorithms +} +``` + +### RB-005: Bundle Verifier + +Implement verification with Attestor integration: + +```csharp +public interface IBundleVerifier +{ + /// + /// Verifies a bundle's DSSE signature and integrity. + /// + Task VerifyAsync( + string bundleDirectory, + VerificationOptions options, + CancellationToken ct = default); +} + +public sealed record VerificationOptions +{ + public string? AttestorUrl { get; init; } + public bool RequireRekorProof { get; init; } = false; + public IReadOnlyList? TrustedKeyIds { get; init; } +} + +public sealed record BundleVerificationResult +{ + public required bool IsValid { get; init; } + public required string BundleVersion { get; init; } + public required DateTimeOffset SignedAt { get; init; } + public required string SignerKeyId { get; init; } + public string? RekorLogId { get; init; } + public IReadOnlyList? ValidationErrors { get; init; } +} + +public sealed class BundleVerifier : IBundleVerifier +{ + private readonly IAttestorClient _attestorClient; + + // Implementation notes: + // - Verify DSSE envelope signature + // - Verify rules file SHA-256 matches manifest + // - Optionally verify Rekor transparency log entry + // - Support offline verification (no network calls) +} +``` + +### RB-006: CLI Bundle Create Command + +Add CLI command for bundle creation: + +```bash +stella secrets bundle create \ + --output ./bundles/2026.01 \ + --bundle-id secrets.ruleset \ + --version 2026.01 \ + --rules ./rules/*.json \ + --sign \ + --key-id stellaops-secrets-signer +``` + +Implementation in `src/Cli/StellaOps.Cli/Commands/Secrets/`: + +```csharp +[Command("secrets bundle create")] +public class BundleCreateCommand +{ + [Option("--output", Required = true)] + public string OutputDirectory { get; set; } + + [Option("--bundle-id", Required = true)] + public string BundleId { get; set; } + + [Option("--version", Required = true)] + public string Version { get; set; } + + [Option("--rules", Required = true)] + public IReadOnlyList RuleFiles { get; set; } + + [Option("--sign")] + public bool Sign { get; set; } + + [Option("--key-id")] + public string? KeyId { get; set; } +} +``` + +### RB-007: CLI Bundle Verify Command + +Add CLI command for bundle verification: + +```bash +stella secrets bundle verify \ + --bundle ./bundles/2026.01 \ + --attestor-url http://attestor.local \ + --require-rekor +``` + +```csharp +[Command("secrets bundle verify")] +public class BundleVerifyCommand +{ + [Option("--bundle", Required = true)] + public string BundleDirectory { get; set; } + + [Option("--attestor-url")] + public string? AttestorUrl { get; set; } + + [Option("--require-rekor")] + public bool RequireRekor { get; set; } +} +``` + +### RB-008: Analyzer Host Integration + +Update SecretsAnalyzerHost to verify bundles on startup: + +```csharp +public sealed class SecretsAnalyzerHost : IHostedService +{ + public async Task StartAsync(CancellationToken ct) + { + var bundlePath = _options.RulesetPath; + + // Verify bundle integrity + var verification = await _verifier.VerifyAsync(bundlePath, new VerificationOptions + { + RequireRekorProof = _options.RequireSignatureVerification + }, ct); + + if (!verification.IsValid) + { + _logger.LogError("Bundle verification failed: {Errors}", + string.Join(", ", verification.ValidationErrors ?? [])); + + if (_options.FailOnInvalidBundle) + throw new InvalidOperationException("Secret detection bundle verification failed"); + + return; // Analyzer disabled + } + + _logger.LogInformation( + "SecretsAnalyzerHost: Loaded bundle {Version} signed by {KeyId} with {Count} rules", + verification.BundleVersion, + verification.SignerKeyId, + _ruleset.Rules.Length); + } +} +``` + +### RB-009: Default Rule Definitions + +Create initial rule set in `offline/rules/secrets/sources/`: + +| File | Rule ID | Description | +|------|---------|-------------| +| `aws-access-key.json` | `stellaops.secrets.aws-access-key` | AWS Access Key ID | +| `aws-secret-key.json` | `stellaops.secrets.aws-secret-key` | AWS Secret Access Key | +| `github-pat.json` | `stellaops.secrets.github-pat` | GitHub Personal Access Token | +| `github-app.json` | `stellaops.secrets.github-app` | GitHub App Token (ghs_, ghp_) | +| `gitlab-pat.json` | `stellaops.secrets.gitlab-pat` | GitLab Personal Access Token | +| `azure-storage.json` | `stellaops.secrets.azure-storage-key` | Azure Storage Account Key | +| `gcp-service-account.json` | `stellaops.secrets.gcp-service-account` | GCP Service Account JSON | +| `private-key-rsa.json` | `stellaops.secrets.private-key-rsa` | RSA Private Key (PEM) | +| `private-key-ec.json` | `stellaops.secrets.private-key-ec` | EC Private Key (PEM) | +| `private-key-openssh.json` | `stellaops.secrets.private-key-openssh` | OpenSSH Private Key | +| `jwt.json` | `stellaops.secrets.jwt` | JSON Web Token | +| `slack-token.json` | `stellaops.secrets.slack-token` | Slack API Token | +| `stripe-key.json` | `stellaops.secrets.stripe-key` | Stripe API Key | +| `sendgrid-key.json` | `stellaops.secrets.sendgrid-key` | SendGrid API Key | +| `generic-api-key.json` | `stellaops.secrets.generic-api-key` | Generic high-entropy API key | + +### RB-010: Initial Bundle Build + +Create the signed 2026.01 bundle: + +``` +offline/rules/secrets/2026.01/ +├── secrets.ruleset.manifest.json +├── secrets.ruleset.rules.jsonl +└── secrets.ruleset.dsse.json +``` + +Build script in `.gitea/scripts/build/build-secrets-bundle.sh`: + +```bash +#!/bin/bash +set -euo pipefail + +VERSION="${1:-2026.01}" +OUTPUT_DIR="offline/rules/secrets/${VERSION}" + +stella secrets bundle create \ + --output "${OUTPUT_DIR}" \ + --bundle-id secrets.ruleset \ + --version "${VERSION}" \ + --rules offline/rules/secrets/sources/*.json \ + --sign \ + --key-id stellaops-secrets-signer +``` + +### RB-011: Tests + +Unit tests: +- `BundleBuilderTests.cs` - Bundle creation and validation +- `BundleVerifierTests.cs` - Signature verification +- `RuleValidatorTests.cs` - Rule schema validation + +Integration tests: +- `BundleRoundtripTests.cs` - Create, sign, verify cycle +- `CliCommandTests.cs` - CLI command execution + +### RB-012: Documentation + +Update `docs/modules/scanner/operations/secret-leak-detection.md`: +- Add bundle creation workflow +- Document verification process +- Add troubleshooting for signature failures + +Create `docs/modules/scanner/operations/secrets-bundle-rotation.md`: +- Bundle versioning strategy +- Rotation procedures +- Rollback instructions + +## Bundle Directory Structure + +``` +offline/rules/secrets/ +├── sources/ # Source rule definitions (not distributed) +│ ├── aws-access-key.json +│ ├── aws-secret-key.json +│ └── ... +├── 2026.01/ # Signed release bundle +│ ├── secrets.ruleset.manifest.json +│ ├── secrets.ruleset.rules.jsonl +│ └── secrets.ruleset.dsse.json +└── latest -> 2026.01 # Symlink to latest stable +``` + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| NDJSON for rules | Streaming parse, git-friendly, easy validation | +| Sign manifest (not rules) | Manifest includes rules digest; smaller signature payload | +| Optional Rekor verification | Supports air-gapped deployments | +| Symlink for latest | Simple upgrade path, atomic switch | +| Source rules in repo | Version control, review process for rule changes | + +## Execution Log + +| Date | Action | Notes | +|------|--------|-------| +| 2026-01-04 | Sprint created | Part of secret leak detection implementation | +| 2026-01-04 | RB-001 to RB-010 | Implemented bundle infrastructure, signing, verification, CLI commands | +| 2026-01-04 | RB-011 | Fixed and validated 37 unit tests for bundle system | +| 2026-01-04 | RB-012 | Updated secret-leak-detection.md, created secrets-bundle-rotation.md | +| 2026-01-04 | Sprint completed | All 12 tasks DONE | + diff --git a/docs/implplan/SPRINT_20260104_004_POLICY_secret_dsl_integration.md b/docs/implplan/SPRINT_20260104_004_POLICY_secret_dsl_integration.md new file mode 100644 index 000000000..b1e22fcdb --- /dev/null +++ b/docs/implplan/SPRINT_20260104_004_POLICY_secret_dsl_integration.md @@ -0,0 +1,543 @@ +# Sprint 20260104_004_POLICY - Secret Leak Detection Policy DSL Integration + +## Topic & Scope + +Extend the Policy Engine and stella-dsl with `secret.*` predicates to enable policy-driven decisions on secret leak findings. This sprint delivers the policy integration layer. + +**Key deliverables:** +1. **Policy Predicates**: `secret.hasFinding()`, `secret.bundle.version()`, `secret.match.count()`, `secret.mask.applied` +2. **Evidence Binding**: Connect SecretLeakEvidence to policy evaluation context +3. **Example Policies**: Sample policies for common secret blocking/warning scenarios +4. **Policy Validation**: Schema updates for secret-related predicates + +**Working directory:** `src/Policy/`, `src/PolicyDsl/` + +## Dependencies & Concurrency + +- **Depends on**: Sprint 20260104_002 (Core Analyzer - SecretLeakEvidence model) +- **Parallel with**: Sprint 20260104_003 (Rule Bundles) +- **Required by**: Sprint 20260104_005 (Offline Kit Integration) + +## Documentation Prerequisites + +- docs/modules/policy/architecture.md +- docs/policy/dsl.md +- docs/modules/policy/secret-leak-detection-readiness.md +- CLAUDE.md Section 8 (Code Quality) + +## Delivery Tracker + +| # | Task ID | Status | Key dependency / next step | Owners | Task Definition | +| --- | --- | --- | --- | --- | --- | +| 1 | PSD-001 | TODO | None | Policy Guild | Define ISecretEvidenceProvider interface | +| 2 | PSD-002 | TODO | PSD-001 | Policy Guild | Implement SecretEvidenceContext binding | +| 3 | PSD-003 | TODO | None | Policy Guild | Add secret.hasFinding() predicate | +| 4 | PSD-004 | TODO | None | Policy Guild | Add secret.bundle.version() predicate | +| 5 | PSD-005 | TODO | None | Policy Guild | Add secret.match.count() predicate | +| 6 | PSD-006 | TODO | None | Policy Guild | Add secret.mask.applied predicate | +| 7 | PSD-007 | TODO | None | Policy Guild | Add secret.path.allowlist() predicate | +| 8 | PSD-008 | TODO | PSD-003-007 | Policy Guild | Register predicates in PolicyDslRegistry | +| 9 | PSD-009 | TODO | PSD-008 | Policy Guild | Update DSL schema validation | +| 10 | PSD-010 | TODO | PSD-008 | Policy Guild | Create example policy templates | +| 11 | PSD-011 | TODO | All | Policy Guild | Add unit and integration tests | +| 12 | PSD-012 | TODO | All | Docs Guild | Update policy/dsl.md documentation | + +## Task Details + +### PSD-001: Secret Evidence Provider Interface + +Define the interface for secret evidence access: + +```csharp +/// +/// Provides secret leak evidence for policy evaluation. +/// +public interface ISecretEvidenceProvider +{ + /// + /// Gets all secret findings for the current evaluation context. + /// + IReadOnlyList GetFindings(); + + /// + /// Gets the active rule bundle metadata. + /// + SecretBundleMetadata? GetBundleMetadata(); + + /// + /// Checks if masking was successfully applied to all findings. + /// + bool IsMaskingApplied(); +} + +public sealed record SecretBundleMetadata( + string BundleId, + string Version, + DateTimeOffset SignedAt, + int RuleCount); +``` + +Location: `src/Policy/__Libraries/StellaOps.Policy/Secrets/` + +### PSD-002: Evidence Context Binding + +Bind secret evidence to the policy evaluation context: + +```csharp +public sealed class SecretEvidenceContext +{ + private readonly ISecretEvidenceProvider _provider; + + public SecretEvidenceContext(ISecretEvidenceProvider provider) + { + _provider = provider; + } + + public IReadOnlyList Findings => _provider.GetFindings(); + public SecretBundleMetadata? Bundle => _provider.GetBundleMetadata(); + public bool MaskingApplied => _provider.IsMaskingApplied(); +} + +// Integration with PolicyEvaluationContext +public sealed class PolicyEvaluationContext +{ + // ... existing properties ... + + public SecretEvidenceContext? Secrets { get; init; } +} +``` + +### PSD-003: hasFinding Predicate + +Implement the `secret.hasFinding()` predicate: + +```csharp +/// +/// Returns true if any secret finding matches the filter criteria. +/// +/// +/// secret.hasFinding() // Any finding +/// secret.hasFinding(severity: "high") // High severity +/// secret.hasFinding(ruleId: "stellaops.secrets.aws-*") // AWS rules (glob) +/// secret.hasFinding(severity: "high", confidence: "high") // Both filters +/// +[DslPredicate("secret.hasFinding")] +public sealed class SecretHasFindingPredicate : IPolicyPredicate +{ + public bool Evaluate( + PolicyEvaluationContext context, + IReadOnlyDictionary? args) + { + var findings = context.Secrets?.Findings ?? []; + if (findings.Count == 0) return false; + + var ruleIdPattern = args?.GetValueOrDefault("ruleId") as string; + var severity = args?.GetValueOrDefault("severity") as string; + var confidence = args?.GetValueOrDefault("confidence") as string; + + return findings.Any(f => + MatchesRuleId(f.RuleId, ruleIdPattern) && + MatchesSeverity(f.Severity, severity) && + MatchesConfidence(f.Confidence, confidence)); + } + + private static bool MatchesRuleId(string ruleId, string? pattern) + { + if (string.IsNullOrEmpty(pattern)) return true; + if (pattern.EndsWith("*")) + return ruleId.StartsWith(pattern[..^1], StringComparison.Ordinal); + return ruleId.Equals(pattern, StringComparison.Ordinal); + } +} +``` + +Location: `src/Policy/__Libraries/StellaOps.Policy/Predicates/Secret/` + +### PSD-004: bundle.version Predicate + +Implement the `secret.bundle.version()` predicate: + +```csharp +/// +/// Returns true if the active bundle meets or exceeds the required version. +/// +/// +/// secret.bundle.version("2026.01") // At least 2026.01 +/// +[DslPredicate("secret.bundle.version")] +public sealed class SecretBundleVersionPredicate : IPolicyPredicate +{ + public bool Evaluate( + PolicyEvaluationContext context, + IReadOnlyDictionary? args) + { + var requiredVersion = args?.GetValueOrDefault("requiredVersion") as string + ?? throw new PolicyEvaluationException("secret.bundle.version requires requiredVersion argument"); + + var bundle = context.Secrets?.Bundle; + if (bundle == null) return false; + + return CompareVersions(bundle.Version, requiredVersion) >= 0; + } + + private static int CompareVersions(string current, string required) + { + // Simple calendar version comparison (YYYY.MM format) + return string.Compare(current, required, StringComparison.Ordinal); + } +} +``` + +### PSD-005: match.count Predicate + +Implement the `secret.match.count()` predicate: + +```csharp +/// +/// Returns the count of findings matching the filter criteria. +/// +/// +/// secret.match.count() > 0 // Any findings +/// secret.match.count(ruleId: "stellaops.secrets.aws-*") >= 5 // Many AWS findings +/// +[DslPredicate("secret.match.count")] +public sealed class SecretMatchCountPredicate : IPolicyPredicate +{ + public int Evaluate( + PolicyEvaluationContext context, + IReadOnlyDictionary? args) + { + var findings = context.Secrets?.Findings ?? []; + + var ruleIdPattern = args?.GetValueOrDefault("ruleId") as string; + + return findings.Count(f => MatchesRuleId(f.RuleId, ruleIdPattern)); + } +} +``` + +### PSD-006: mask.applied Predicate + +Implement the `secret.mask.applied` predicate: + +```csharp +/// +/// Returns true if masking was successfully applied to all findings. +/// +/// +/// secret.mask.applied // Verify masking succeeded +/// +[DslPredicate("secret.mask.applied")] +public sealed class SecretMaskAppliedPredicate : IPolicyPredicate +{ + public bool Evaluate( + PolicyEvaluationContext context, + IReadOnlyDictionary? args) + { + return context.Secrets?.MaskingApplied ?? true; // Default true if no findings + } +} +``` + +### PSD-007: path.allowlist Predicate + +Implement the `secret.path.allowlist()` predicate for false positive suppression: + +```csharp +/// +/// Returns true if all findings are in paths matching the allowlist patterns. +/// +/// +/// secret.path.allowlist(["**/test/**", "**/fixtures/**"]) // Ignore test files +/// +[DslPredicate("secret.path.allowlist")] +public sealed class SecretPathAllowlistPredicate : IPolicyPredicate +{ + public bool Evaluate( + PolicyEvaluationContext context, + IReadOnlyDictionary? args) + { + var patterns = args?.GetValueOrDefault("patterns") as IReadOnlyList; + if (patterns == null || patterns.Count == 0) + throw new PolicyEvaluationException("secret.path.allowlist requires patterns argument"); + + var findings = context.Secrets?.Findings ?? []; + if (findings.Count == 0) return true; + + return findings.All(f => patterns.Any(p => GlobMatcher.IsMatch(f.FilePath, p))); + } +} +``` + +### PSD-008: Predicate Registration + +Register predicates in the DSL registry: + +```csharp +public static class SecretPredicateRegistration +{ + public static void RegisterSecretPredicates(this PolicyDslRegistry registry) + { + registry.RegisterPredicate("secret.hasFinding"); + registry.RegisterPredicate("secret.bundle.version"); + registry.RegisterPredicate("secret.match.count"); + registry.RegisterPredicate("secret.mask.applied"); + registry.RegisterPredicate("secret.path.allowlist"); + } +} +``` + +### PSD-009: DSL Schema Validation + +Update the DSL schema to include secret predicates: + +```json +{ + "predicates": { + "secret.hasFinding": { + "description": "Returns true if any secret finding matches the filter", + "arguments": { + "ruleId": { "type": "string", "optional": true, "description": "Rule ID pattern (supports * glob)" }, + "severity": { "type": "string", "optional": true, "enum": ["low", "medium", "high", "critical"] }, + "confidence": { "type": "string", "optional": true, "enum": ["low", "medium", "high"] } + }, + "returns": "boolean" + }, + "secret.bundle.version": { + "description": "Returns true if the active bundle meets or exceeds the required version", + "arguments": { + "requiredVersion": { "type": "string", "required": true, "description": "Minimum required version (YYYY.MM format)" } + }, + "returns": "boolean" + }, + "secret.match.count": { + "description": "Returns the count of findings matching the filter", + "arguments": { + "ruleId": { "type": "string", "optional": true, "description": "Rule ID pattern (supports * glob)" } + }, + "returns": "integer" + }, + "secret.mask.applied": { + "description": "Returns true if masking was successfully applied to all findings", + "arguments": {}, + "returns": "boolean" + }, + "secret.path.allowlist": { + "description": "Returns true if all findings are in paths matching the allowlist", + "arguments": { + "patterns": { "type": "array", "items": { "type": "string" }, "required": true } + }, + "returns": "boolean" + } + } +} +``` + +### PSD-010: Example Policy Templates + +Create example policies in `docs/modules/policy/examples/`: + +**secret-blocker.stella** - Block high-severity secrets: +```dsl +policy "Secret Leak Guard" syntax "stella-dsl@1" { + metadata { + description = "Block high-confidence secret leaks in production scans" + tags = ["secrets", "compliance", "security"] + } + + rule block_critical priority 100 { + when secret.hasFinding(severity: "critical") + then escalate to "block"; + because "Critical secret leak detected - deployment blocked"; + } + + rule block_high_confidence priority 90 { + when secret.hasFinding(severity: "high", confidence: "high") + then escalate to "block"; + because "High severity secret leak with high confidence detected"; + } + + rule warn_medium priority 50 { + when secret.hasFinding(severity: "medium") + then warn message "Medium severity secret detected - review required"; + } + + rule require_current_bundle priority 10 { + when not secret.bundle.version("2026.01") + then warn message "Secret detection bundle is out of date"; + } +} +``` + +**secret-allowlist.stella** - Suppress test file findings: +```dsl +policy "Secret Allowlist" syntax "stella-dsl@1" { + metadata { + description = "Suppress false positives in test fixtures" + tags = ["secrets", "testing"] + } + + rule allow_test_fixtures priority 200 { + when secret.path.allowlist([ + "**/test/**", + "**/tests/**", + "**/fixtures/**", + "**/__fixtures__/**", + "**/testdata/**" + ]) + then annotate decision.notes := "Findings in test paths - suppressed"; + else continue; + } + + rule allow_examples priority 190 { + when secret.path.allowlist([ + "**/examples/**", + "**/samples/**", + "**/docs/**" + ]) + and secret.hasFinding(confidence: "low") + then annotate decision.notes := "Low confidence findings in example paths"; + else continue; + } +} +``` + +**secret-threshold.stella** - Threshold-based blocking: +```dsl +policy "Secret Threshold Guard" syntax "stella-dsl@1" { + metadata { + description = "Block scans exceeding secret finding thresholds" + tags = ["secrets", "thresholds"] + } + + rule excessive_secrets priority 80 { + when secret.match.count() > 50 + then escalate to "block"; + because "Excessive number of secret findings (>50) - likely misconfigured scan"; + } + + rule many_aws_secrets priority 70 { + when secret.match.count(ruleId: "stellaops.secrets.aws-*") > 10 + then escalate to "review"; + because "Multiple AWS credentials detected - security review required"; + } +} +``` + +### PSD-011: Tests + +Unit tests in `src/Policy/__Tests/StellaOps.Policy.Tests/Predicates/Secret/`: + +``` +├── SecretHasFindingPredicateTests.cs +│ - Test empty findings returns false +│ - Test matching severity filter +│ - Test matching confidence filter +│ - Test ruleId glob matching +│ - Test combined filters +├── SecretBundleVersionPredicateTests.cs +│ - Test version comparison +│ - Test missing bundle returns false +│ - Test exact version match +├── SecretMatchCountPredicateTests.cs +│ - Test empty findings returns 0 +│ - Test count with filter +│ - Test count without filter +├── SecretMaskAppliedPredicateTests.cs +│ - Test masking applied +│ - Test masking not applied +│ - Test default for no findings +├── SecretPathAllowlistPredicateTests.cs +│ - Test glob pattern matching +│ - Test multiple patterns +│ - Test no matching patterns +└── PolicyEvaluationIntegrationTests.cs + - Test full policy evaluation with secrets + - Test policy chaining + - Test decision propagation +``` + +### PSD-012: Documentation + +Update `docs/policy/dsl.md` with new predicates section: + +```markdown +## Secret Leak Detection Predicates + +The following predicates are available for secret leak detection policy rules: + +### secret.hasFinding(ruleId?, severity?, confidence?) + +Returns `true` if any secret finding matches the specified filters. + +**Arguments:** +- `ruleId` (string, optional): Rule ID pattern with optional `*` glob suffix +- `severity` (string, optional): One of `low`, `medium`, `high`, `critical` +- `confidence` (string, optional): One of `low`, `medium`, `high` + +**Example:** +```dsl +when secret.hasFinding(severity: "high", confidence: "high") +``` + +### secret.bundle.version(requiredVersion) + +Returns `true` if the active rule bundle version meets or exceeds the required version. + +**Arguments:** +- `requiredVersion` (string, required): Minimum version in `YYYY.MM` format + +**Example:** +```dsl +when not secret.bundle.version("2026.01") +then warn message "Bundle out of date"; +``` + +### secret.match.count(ruleId?) + +Returns the integer count of findings matching the optional rule ID filter. + +**Example:** +```dsl +when secret.match.count() > 50 +``` + +### secret.mask.applied + +Returns `true` if payload masking was successfully applied to all findings. + +**Example:** +```dsl +when not secret.mask.applied +then escalate to "block"; +because "Masking failed - secrets may be exposed"; +``` + +### secret.path.allowlist(patterns) + +Returns `true` if all findings are in file paths matching at least one allowlist pattern. + +**Arguments:** +- `patterns` (array of strings, required): Glob patterns for allowed paths + +**Example:** +```dsl +when secret.path.allowlist(["**/test/**", "**/fixtures/**"]) +``` +``` + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Glob patterns for ruleId | Simple, familiar syntax for rule filtering | +| YYYY.MM version format | Matches bundle versioning convention | +| Default true for mask.applied with no findings | Conservative - don't fail on clean scans | +| Path allowlist as AND | All findings must be in allowed paths to pass | + +## Execution Log + +| Date | Action | Notes | +|------|--------|-------| +| 2026-01-04 | Sprint created | Part of secret leak detection implementation | + diff --git a/docs/implplan/SPRINT_20260104_005_AIRGAP_secret_offline_kit.md b/docs/implplan/SPRINT_20260104_005_AIRGAP_secret_offline_kit.md new file mode 100644 index 000000000..7f815c5d2 --- /dev/null +++ b/docs/implplan/SPRINT_20260104_005_AIRGAP_secret_offline_kit.md @@ -0,0 +1,591 @@ +# Sprint 20260104_005_AIRGAP - Secret Detection Offline Kit Integration + +## Topic & Scope + +Integrate secret detection rule bundles with the Offline Kit infrastructure for air-gapped deployments. This sprint ensures complete offline parity for secret leak detection. + +**Key deliverables:** +1. **Bundle Distribution**: Include signed bundles in Offline Kit exports +2. **Import Workflow**: Bundle import and verification scripts +3. **Attestor Mirror**: Local verification support without internet +4. **CI/CD Integration**: Automated bundle inclusion in releases +5. **Upgrade Path**: Bundle rotation procedures for offline environments + +**Working directory:** `src/AirGap/`, `devops/offline/`, `offline/rules/secrets/` + +## Dependencies & Concurrency + +- **Depends on**: Sprint 20260104_002 (Core Analyzer), Sprint 20260104_003 (Rule Bundles) +- **Parallel with**: Sprint 20260104_004 (Policy DSL) +- **Blocks**: Production deployment of secret leak detection + +## Documentation Prerequisites + +- docs/24_OFFLINE_KIT.md +- docs/modules/airgap/airgap-mode.md +- docs/modules/scanner/operations/secret-leak-detection.md +- CLAUDE.md Section 8 (Determinism) + +## Delivery Tracker + +| # | Task ID | Status | Key dependency / next step | Owners | Task Definition | +| --- | --- | --- | --- | --- | --- | +| 1 | OKS-001 | TODO | None | AirGap Guild | Update Offline Kit manifest schema for rules | +| 2 | OKS-002 | TODO | OKS-001 | AirGap Guild | Add secrets bundle to BundleBuilder | +| 3 | OKS-003 | TODO | OKS-002 | AirGap Guild | Create bundle verification in Importer | +| 4 | OKS-004 | TODO | None | AirGap Guild | Add Attestor mirror support for bundle verification | +| 5 | OKS-005 | TODO | OKS-003 | AirGap Guild | Create bundle installation script | +| 6 | OKS-006 | TODO | OKS-005 | AirGap Guild | Add bundle rotation/upgrade workflow | +| 7 | OKS-007 | TODO | None | CI/CD Guild | Add bundle to release workflow | +| 8 | OKS-008 | TODO | All | AirGap Guild | Add integration tests for offline flow | +| 9 | OKS-009 | TODO | All | Docs Guild | Update offline kit documentation | +| 10 | OKS-010 | TODO | All | DevOps Guild | Update Helm charts for bundle mounting | + +## Task Details + +### OKS-001: Manifest Schema Update + +Update the Offline Kit manifest to include rule bundles: + +```json +{ + "version": "2.1", + "created": "2026-01-04T00:00:00Z", + "components": { + "advisory": { ... }, + "policy": { ... }, + "vex": { ... }, + "rules": { + "secrets": { + "bundleId": "secrets.ruleset", + "version": "2026.01", + "path": "rules/secrets/2026.01", + "files": [ + { + "name": "secrets.ruleset.manifest.json", + "sha256": "abc123..." + }, + { + "name": "secrets.ruleset.rules.jsonl", + "sha256": "def456..." + }, + { + "name": "secrets.ruleset.dsse.json", + "sha256": "ghi789..." + } + ], + "signature": { + "keyId": "stellaops-secrets-signer", + "verifiedAt": "2026-01-04T00:00:00Z" + } + } + } + } +} +``` + +Location: `src/AirGap/__Libraries/StellaOps.AirGap.Bundle/Schemas/` + +### OKS-002: Bundle Builder Extension + +Extend BundleBuilder to include secrets rule bundles: + +```csharp +public sealed class SnapshotBundleBuilder +{ + // Add secrets bundle extraction + public async Task BuildAsync( + BundleBuildContext context, + CancellationToken ct = default) + { + // ... existing extractors ... + + // Add secrets rules extractor + await ExtractSecretsRulesAsync(context, ct); + + return result; + } + + private async Task ExtractSecretsRulesAsync( + BundleBuildContext context, + CancellationToken ct) + { + var sourcePath = _options.SecretsRulesBundlePath; + if (string.IsNullOrEmpty(sourcePath) || !Directory.Exists(sourcePath)) + { + _logger.LogWarning("Secrets rules bundle not found at {Path}", sourcePath); + return; + } + + var targetPath = Path.Combine(context.OutputPath, "rules", "secrets"); + Directory.CreateDirectory(targetPath); + + // Copy bundle files + foreach (var file in Directory.GetFiles(sourcePath, "secrets.ruleset.*")) + { + var targetFile = Path.Combine(targetPath, Path.GetFileName(file)); + await CopyWithIntegrityAsync(file, targetFile, ct); + } + + // Add to manifest + context.Manifest.Rules["secrets"] = new RuleBundleManifest + { + BundleId = "secrets.ruleset", + Version = await ReadBundleVersionAsync(sourcePath, ct), + Path = "rules/secrets" + }; + } +} +``` + +Location: `src/AirGap/__Libraries/StellaOps.AirGap.Bundle/Services/` + +### OKS-003: Importer Verification + +Add bundle verification to the Offline Kit importer: + +```csharp +public sealed class OfflineKitImporter +{ + public async Task ImportAsync( + string kitPath, + ImportOptions options, + CancellationToken ct = default) + { + // ... existing imports ... + + // Import and verify secrets rules + if (manifest.Rules.TryGetValue("secrets", out var secretsBundle)) + { + await ImportSecretsRulesAsync(kitPath, secretsBundle, options, ct); + } + + return result; + } + + private async Task ImportSecretsRulesAsync( + string kitPath, + RuleBundleManifest bundle, + ImportOptions options, + CancellationToken ct) + { + var sourcePath = Path.Combine(kitPath, bundle.Path); + var targetPath = _options.SecretsRulesInstallPath; + + // Verify bundle integrity + var verifier = _serviceProvider.GetRequiredService(); + var verification = await verifier.VerifyAsync(sourcePath, new VerificationOptions + { + AttestorUrl = options.AttestorMirrorUrl, + RequireRekorProof = options.RequireRekorProof + }, ct); + + if (!verification.IsValid) + { + throw new ImportException($"Secrets bundle verification failed: {string.Join(", ", verification.ValidationErrors ?? [])}"); + } + + // Install bundle + await InstallBundleAsync(sourcePath, targetPath, ct); + + _logger.LogInformation( + "Installed secrets bundle {Version} signed by {KeyId}", + verification.BundleVersion, + verification.SignerKeyId); + } +} +``` + +### OKS-004: Attestor Mirror Support + +Configure Attestor mirror for offline bundle verification: + +```csharp +public sealed class OfflineAttestorClient : IAttestorClient +{ + private readonly string _mirrorPath; + + public OfflineAttestorClient(string mirrorPath) + { + _mirrorPath = mirrorPath; + } + + public async Task VerifyDsseAsync( + DsseEnvelope envelope, + VerifyOptions options, + CancellationToken ct = default) + { + // Load mirrored certificate chain + var chainPath = Path.Combine(_mirrorPath, "certs", options.KeyId + ".pem"); + if (!File.Exists(chainPath)) + { + return VerificationResult.Failed($"Certificate not found in mirror: {options.KeyId}"); + } + + var chain = await LoadCertificateChainAsync(chainPath, ct); + + // Verify signature locally + var result = await _dsseVerifier.VerifyAsync(envelope, chain, ct); + + // Optionally verify against mirrored Rekor entries + if (options.RequireRekorProof) + { + var rekorPath = Path.Combine(_mirrorPath, "rekor", envelope.PayloadDigest + ".json"); + if (!File.Exists(rekorPath)) + { + return VerificationResult.Failed("Rekor entry not found in mirror"); + } + + result = result.WithRekorEntry(await LoadRekorEntryAsync(rekorPath, ct)); + } + + return result; + } +} +``` + +### OKS-005: Installation Script + +Create bundle installation script for operators: + +**`devops/offline/scripts/install-secrets-bundle.sh`:** +```bash +#!/bin/bash +set -euo pipefail + +BUNDLE_PATH="${1:-/mnt/offline-kit/rules/secrets}" +INSTALL_PATH="${2:-/opt/stellaops/plugins/scanner/analyzers/secrets}" +ATTESTOR_MIRROR="${3:-/mnt/offline-kit/attestor-mirror}" + +echo "Installing secrets bundle from ${BUNDLE_PATH}" + +# Verify bundle before installation +export STELLA_ATTESTOR_URL="file://${ATTESTOR_MIRROR}" + +if ! stella secrets bundle verify --bundle "${BUNDLE_PATH}" --require-rekor; then + echo "ERROR: Bundle verification failed" >&2 + exit 1 +fi + +# Create installation directory +mkdir -p "${INSTALL_PATH}" + +# Install bundle files +cp -v "${BUNDLE_PATH}"/secrets.ruleset.* "${INSTALL_PATH}/" + +# Set permissions +chmod 640 "${INSTALL_PATH}"/secrets.ruleset.* +chown stellaops:stellaops "${INSTALL_PATH}"/secrets.ruleset.* + +# Verify installation +INSTALLED_VERSION=$(jq -r '.version' "${INSTALL_PATH}/secrets.ruleset.manifest.json") +echo "Successfully installed secrets bundle version ${INSTALLED_VERSION}" + +echo "" +echo "To activate, restart Scanner Worker:" +echo " systemctl restart stellaops-scanner-worker" +echo "" +echo "Or with Kubernetes:" +echo " kubectl rollout restart deployment/scanner-worker" +``` + +### OKS-006: Bundle Rotation Workflow + +Document and implement bundle upgrade procedure: + +**Upgrade Workflow:** + +1. **Pre-upgrade Verification** + ```bash + # Verify new bundle + stella secrets bundle verify --bundle /path/to/new-bundle + + # Compare with current + CURRENT=$(jq -r '.version' /opt/stellaops/.../secrets.ruleset.manifest.json) + NEW=$(jq -r '.version' /path/to/new-bundle/secrets.ruleset.manifest.json) + echo "Upgrading from ${CURRENT} to ${NEW}" + ``` + +2. **Backup Current Bundle** + ```bash + BACKUP_DIR="/opt/stellaops/backups/secrets-bundles/$(date +%Y%m%d)" + mkdir -p "${BACKUP_DIR}" + cp -a /opt/stellaops/plugins/scanner/analyzers/secrets/* "${BACKUP_DIR}/" + ``` + +3. **Install New Bundle** + ```bash + ./install-secrets-bundle.sh /path/to/new-bundle + ``` + +4. **Rolling Restart** + ```bash + # Kubernetes + kubectl rollout restart deployment/scanner-worker --namespace stellaops + + # Systemd + systemctl restart stellaops-scanner-worker + ``` + +5. **Verify Upgrade** + ```bash + # Check logs for new version + kubectl logs -l app=scanner-worker --tail=100 | grep "SecretsAnalyzerHost" + ``` + +**Rollback Procedure:** +```bash +# Restore backup +cp -a "${BACKUP_DIR}"/* /opt/stellaops/plugins/scanner/analyzers/secrets/ + +# Restart workers +kubectl rollout restart deployment/scanner-worker +``` + +### OKS-007: Release Workflow Integration + +Add secrets bundle to CI/CD release pipeline: + +**`.gitea/workflows/release-offline-kit.yml`:** +```yaml +jobs: + build-offline-kit: + steps: + - name: Build secrets bundle + run: | + stella secrets bundle create \ + --output ./offline-kit/rules/secrets/${VERSION} \ + --bundle-id secrets.ruleset \ + --version ${VERSION} \ + --rules ./offline/rules/secrets/sources/*.json \ + --sign \ + --key-id ${SECRETS_SIGNER_KEY_ID} + + - name: Include in offline kit + run: | + # Bundle is automatically included via BundleBuilder + + - name: Verify bundle in kit + run: | + stella secrets bundle verify \ + --bundle ./offline-kit/rules/secrets/${VERSION} +``` + +**Add to `.gitea/scripts/build/build-offline-kit.sh`:** +```bash +# Build and sign secrets bundle +echo "Building secrets rule bundle..." +stella secrets bundle create \ + --output "${OUTPUT_DIR}/rules/secrets/${BUNDLE_VERSION}" \ + --bundle-id secrets.ruleset \ + --version "${BUNDLE_VERSION}" \ + --rules offline/rules/secrets/sources/*.json \ + --sign \ + --key-id "${SECRETS_SIGNER_KEY_ID}" +``` + +### OKS-008: Integration Tests + +Create integration tests for offline flow: + +```csharp +[Trait("Category", "Integration")] +public class OfflineSecretsIntegrationTests : IClassFixture +{ + [Fact] + public async Task OfflineKit_IncludesSecretsBundleWithValidSignature() + { + // Arrange + var kit = await _fixture.BuildOfflineKitAsync(); + + // Act + var bundlePath = Path.Combine(kit.Path, "rules", "secrets"); + var verifier = new BundleVerifier(_attestorMirror); + var result = await verifier.VerifyAsync(bundlePath, new VerificationOptions()); + + // Assert + result.IsValid.Should().BeTrue(); + result.BundleVersion.Should().NotBeEmpty(); + } + + [Fact] + public async Task Importer_InstallsAndVerifiesBundle() + { + // Arrange + var kit = await _fixture.BuildOfflineKitAsync(); + var importer = new OfflineKitImporter(_options); + + // Act + var result = await importer.ImportAsync(kit.Path, new ImportOptions + { + AttestorMirrorUrl = _attestorMirrorUrl + }); + + // Assert + result.Success.Should().BeTrue(); + Directory.Exists(_installPath).Should().BeTrue(); + File.Exists(Path.Combine(_installPath, "secrets.ruleset.manifest.json")).Should().BeTrue(); + } + + [Fact] + public async Task Scanner_LoadsBundleFromOfflineInstallation() + { + // Arrange + await ImportOfflineKitAsync(); + + // Act + var host = new SecretsAnalyzerHost(_options, _logger); + await host.StartAsync(CancellationToken.None); + + // Assert + host.IsEnabled.Should().BeTrue(); + host.BundleVersion.Should().Be(_expectedVersion); + } +} +``` + +### OKS-009: Documentation Updates + +Update `docs/24_OFFLINE_KIT.md`: + +```markdown +## Secret Detection Rules + +The Offline Kit includes DSSE-signed rule bundles for secret leak detection. + +### Bundle Contents + +``` +offline-kit/ +├── rules/ +│ └── secrets/ +│ └── 2026.01/ +│ ├── secrets.ruleset.manifest.json # Rule metadata +│ ├── secrets.ruleset.rules.jsonl # Rule definitions +│ └── secrets.ruleset.dsse.json # DSSE signature +└── attestor-mirror/ + ├── certs/ + │ └── stellaops-secrets-signer.pem # Signing certificate + └── rekor/ + └── .json # Transparency log entry +``` + +### Installation + +1. **Verify Bundle** + ```bash + export STELLA_ATTESTOR_URL="file:///mnt/offline-kit/attestor-mirror" + stella secrets bundle verify --bundle /mnt/offline-kit/rules/secrets/2026.01 + ``` + +2. **Install Bundle** + ```bash + ./devops/offline/scripts/install-secrets-bundle.sh \ + /mnt/offline-kit/rules/secrets/2026.01 \ + /opt/stellaops/plugins/scanner/analyzers/secrets + ``` + +3. **Enable Feature** + ```yaml + scanner: + features: + experimental: + secret-leak-detection: true + ``` + +4. **Restart Workers** + ```bash + kubectl rollout restart deployment/scanner-worker + ``` + +### Verification + +Check that the bundle is loaded: +```bash +kubectl logs -l app=scanner-worker --tail=100 | grep SecretsAnalyzerHost +# Expected: SecretsAnalyzerHost: Loaded bundle 2026.01 signed by stellaops-secrets-signer with N rules +``` + +### Bundle Rotation + +See [Secret Bundle Rotation](./modules/scanner/operations/secrets-bundle-rotation.md) for upgrade procedures. +``` + +### OKS-010: Helm Chart Updates + +Update Helm charts for bundle mounting: + +**`devops/helm/stellaops/templates/scanner-worker-deployment.yaml`:** +```yaml +spec: + template: + spec: + volumes: + # Existing volumes... + - name: secrets-rules + {{- if .Values.scanner.secretsRules.persistentVolumeClaim }} + persistentVolumeClaim: + claimName: {{ .Values.scanner.secretsRules.persistentVolumeClaim }} + {{- else }} + configMap: + name: {{ include "stellaops.fullname" . }}-secrets-rules + {{- end }} + containers: + - name: scanner-worker + volumeMounts: + # Existing mounts... + - name: secrets-rules + mountPath: /opt/stellaops/plugins/scanner/analyzers/secrets + readOnly: true +``` + +**`devops/helm/stellaops/values.yaml`:** +```yaml +scanner: + features: + experimental: + secretLeakDetection: false # Enable via override + + secretsRules: + # Use PVC for air-gapped installations + persistentVolumeClaim: "" + # Or use ConfigMap for simple deployments + bundleVersion: "2026.01" +``` + +## Directory Structure + +``` +offline/rules/secrets/ +├── sources/ # Source rule JSON files (not in kit) +│ ├── aws-access-key.json +│ └── ... +├── 2026.01/ # Signed bundle (in kit) +│ ├── secrets.ruleset.manifest.json +│ ├── secrets.ruleset.rules.jsonl +│ └── secrets.ruleset.dsse.json +└── latest -> 2026.01 # Symlink + +devops/offline/ +├── scripts/ +│ ├── install-secrets-bundle.sh # Installation script +│ └── rotate-secrets-bundle.sh # Rotation script +└── templates/ + └── secrets-bundle-pvc.yaml # PVC template for air-gap +``` + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Include Attestor mirror in kit | Enables fully offline verification | +| File:// URL for offline Attestor | Simple, no network required | +| ConfigMap fallback | Simpler for non-air-gapped deployments | +| Symlink for latest | Atomic version switching | + +## Execution Log + +| Date | Action | Notes | +|------|--------|-------| +| 2026-01-04 | Sprint created | Part of secret leak detection implementation | + diff --git a/docs/implplan/SPRINT_20260104_001_BE_adaptive_noise_gating.md b/docs/implplan/archived/2026-01-04-completed-sprints/SPRINT_20260104_001_BE_adaptive_noise_gating.md similarity index 78% rename from docs/implplan/SPRINT_20260104_001_BE_adaptive_noise_gating.md rename to docs/implplan/archived/2026-01-04-completed-sprints/SPRINT_20260104_001_BE_adaptive_noise_gating.md index 09640b315..05d42bac5 100644 --- a/docs/implplan/SPRINT_20260104_001_BE_adaptive_noise_gating.md +++ b/docs/implplan/archived/2026-01-04-completed-sprints/SPRINT_20260104_001_BE_adaptive_noise_gating.md @@ -28,17 +28,17 @@ Implement adaptive noise-gating for vulnerability graphs to reduce alert fatigue | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | NG-001 | TODO | None | Guild | Add ProofStrength enum to StellaOps.Evidence.Core | -| 2 | NG-002 | TODO | NG-001 | Guild | Add ProofStrength field to EvidenceRecord | -| 3 | NG-003 | TODO | None | Guild | Create EdgeSemanticKey and deduplication logic in ReachGraph | -| 4 | NG-004 | TODO | None | Guild | Add StabilityDampingGate to Policy.Engine.Gates | -| 5 | NG-005 | TODO | NG-004 | Guild | Add StabilityDampingOptions with configurable thresholds | -| 6 | NG-006 | TODO | None | Guild | Create DeltaSection enum in VexLens | -| 7 | NG-007 | TODO | NG-006 | Guild | Extend VexDelta with section categorization | -| 8 | NG-008 | TODO | NG-001,NG-003,NG-004,NG-006 | Guild | Create INoiseGate interface and NoiseGateService | -| 9 | NG-009 | TODO | NG-008 | Guild | Add DI registration in VexLensServiceCollectionExtensions | -| 10 | NG-010 | TODO | All | Guild | Add unit tests for all new components | -| 11 | NG-011 | TODO | NG-010 | Guild | Update module AGENTS.md files | +| 1 | NG-001 | DONE | None | Guild | Add ProofStrength enum to StellaOps.Evidence.Core | +| 2 | NG-002 | DONE | NG-001 | Guild | Add ProofStrength field to EvidenceRecord | +| 3 | NG-003 | DONE | None | Guild | Create EdgeSemanticKey and deduplication logic in ReachGraph | +| 4 | NG-004 | DONE | None | Guild | Add StabilityDampingGate to Policy.Engine.Gates | +| 5 | NG-005 | DONE | NG-004 | Guild | Add StabilityDampingOptions with configurable thresholds | +| 6 | NG-006 | DONE | None | Guild | Create DeltaSection enum in VexLens | +| 7 | NG-007 | DONE | NG-006 | Guild | Extend VexDelta with section categorization | +| 8 | NG-008 | DONE | NG-001,NG-003,NG-004,NG-006 | Guild | Create INoiseGate interface and NoiseGateService | +| 9 | NG-009 | DONE | NG-008 | Guild | Add DI registration in VexLensServiceCollectionExtensions | +| 10 | NG-010 | DONE | All | Guild | Add unit tests for all new components | +| 11 | NG-011 | DONE | NG-010 | Guild | Update module AGENTS.md files | ## Task Details @@ -184,4 +184,12 @@ Update module documentation: | Date | Action | Notes | |------|--------|-------| | 2026-01-04 | Sprint created | Based on product advisory review | +| 2026-01-04 | NG-001,NG-002 | Created ProofStrength enum, ProofStrengthExtensions, ProofRecord in StellaOps.Evidence.Models | +| 2026-01-04 | NG-003 | Created EdgeSemanticKey, DeduplicatedEdge, EdgeDeduplicator in StellaOps.ReachGraph.Deduplication | +| 2026-01-04 | NG-004,NG-005 | Created StabilityDampingGate, StabilityDampingOptions in StellaOps.Policy.Engine.Gates | +| 2026-01-04 | NG-006,NG-007 | Created DeltaSection, DeltaEntry, DeltaReport, DeltaReportBuilder in StellaOps.VexLens.Delta | +| 2026-01-04 | NG-008,NG-009 | Created INoiseGate, NoiseGateService, NoiseGateOptions; registered DI in VexLensServiceCollectionExtensions | +| 2026-01-04 | NG-010 | Added StabilityDampingGateTests, NoiseGateServiceTests, DeltaReportBuilderTests | +| 2026-01-04 | NG-011 | Updated VexLens and Policy.Engine AGENTS.md files | +| 2026-01-04 | Sprint complete | All 11 tasks DONE | diff --git a/docs/implplan/archived/SPRINT_20260104_002_FE_noise_gating_delta_ui.md b/docs/implplan/archived/SPRINT_20260104_002_FE_noise_gating_delta_ui.md new file mode 100644 index 000000000..7fe33f80c --- /dev/null +++ b/docs/implplan/archived/SPRINT_20260104_002_FE_noise_gating_delta_ui.md @@ -0,0 +1,223 @@ +# Sprint 20260104_002_FE - Noise-Gating Delta Report UI + +## Topic & Scope + +Implement frontend components to display noise-gating delta reports from the VexLens backend. This sprint composes existing Angular components to minimize new code while providing a complete UI for: + +1. **Delta Report Display**: Show changes between vulnerability graph snapshots +2. **Section-Based Navigation**: Tabs for New, Resolved, ConfidenceUp/Down, PolicyImpact, Damped sections +3. **Gating Statistics**: Edge deduplication rates and verdict damping metrics +4. **Backend API Endpoints**: Expose DeltaReport via VexLens WebService + +**Working directories:** +- `src/Web/StellaOps.Web/src/app/` (frontend) +- `src/VexLens/StellaOps.VexLens.WebService/` (backend API) + +## Dependencies & Concurrency + +- Builds on completed Sprint 20260104_001_BE (backend NoiseGate implementation) +- Reuses existing components: `DeltaSummaryStripComponent`, `TabsComponent`, `GatingExplainerComponent` +- Tasks NG-FE-001 through NG-FE-003 (backend + models) must complete before NG-FE-004+ + +## Existing Components to Reuse + +| Component | Location | Usage | +|-----------|----------|-------| +| `DeltaSummaryStripComponent` | `features/compare/components/` | Overview stats display | +| `TabsComponent` / `TabPanelDirective` | `shared/components/tabs/` | Section navigation | +| `GatingExplainerComponent` | `features/triage/components/gating-explainer/` | Per-entry explanations | +| `DeltaComputeService` patterns | `features/compare/services/` | Signal-based state management | +| `GatingReason`, `DeltaSummary` | `features/triage/models/gating.model.ts` | Existing delta/gating types | +| `VexStatementStatus` | `core/api/vex-hub.models.ts` | VEX status types | +| `BadgeComponent`, `StatCardComponent` | `shared/components/` | Statistics display | + +## Documentation Prerequisites + +- CLAUDE.md (Section 8: Code Quality rules) +- src/Web/StellaOps.Web/README.md +- docs/modules/vexlens/architecture.md + +## Delivery Tracker + +| # | Task ID | Status | Key dependency | Task Definition | +|---|---------|--------|----------------|-----------------| +| 1 | NG-FE-001 | DONE | None | Add delta report API endpoints to VexLens.WebService | +| 2 | NG-FE-002 | DONE | None | Create TypeScript models for noise-gating delta (noise-gating.models.ts) | +| 3 | NG-FE-003 | DONE | NG-FE-002 | Create NoiseGatingApiClient service | +| 4 | NG-FE-004 | DONE | NG-FE-003 | Create NoiseGatingSummaryStripComponent (extends DeltaSummaryStrip) | +| 5 | NG-FE-005 | DONE | NG-FE-003 | Create DeltaEntryCardComponent for individual entries | +| 6 | NG-FE-006 | DONE | NG-FE-004,005 | Create NoiseGatingDeltaReportComponent (container with tabs) | +| 7 | NG-FE-007 | DONE | NG-FE-006 | Create GatingStatisticsCardComponent | +| 8 | NG-FE-008 | DONE | NG-FE-006 | Integrate into vuln-explorer/triage workspace | +| 9 | NG-FE-009 | DONE | All | Update feature module exports and routing | + +## Task Details + +### NG-FE-001: Backend API Endpoints + +Add endpoints to `VexLensEndpointExtensions.cs`: + +```csharp +// Delta computation +POST /api/v1/vexlens/deltas/compute + Body: { fromSnapshotId, toSnapshotId, options } + Returns: DeltaReportResponse + +// Get gated snapshot +GET /api/v1/vexlens/snapshots/{snapshotId}/gated + Returns: GatedGraphSnapshotResponse + +// Get gating statistics +GET /api/v1/vexlens/gating/statistics + Query: tenantId, fromDate, toDate + Returns: GatingStatisticsResponse +``` + +### NG-FE-002: TypeScript Models + +Create `src/app/core/api/noise-gating.models.ts`: + +```typescript +// Match backend DeltaSection enum +export type NoiseGatingDeltaSection = + | 'new' | 'resolved' | 'confidence_up' | 'confidence_down' + | 'policy_impact' | 'damped' | 'evidence_changed'; + +// Match backend DeltaEntry +export interface NoiseGatingDeltaEntry { + section: NoiseGatingDeltaSection; + vulnerabilityId: string; + productKey: string; + fromStatus?: VexStatementStatus; + toStatus?: VexStatementStatus; + fromConfidence?: number; + toConfidence?: number; + justification?: string; + rationaleClass?: string; + summary?: string; + contributingSources?: string[]; + createdAt: string; +} + +// Match backend DeltaReport +export interface NoiseGatingDeltaReport { + reportId: string; + fromSnapshotDigest: string; + toSnapshotDigest: string; + generatedAt: string; + entries: NoiseGatingDeltaEntry[]; + summary: NoiseGatingDeltaSummary; + hasActionableChanges: boolean; +} + +// Summary counts +export interface NoiseGatingDeltaSummary { + totalCount: number; + newCount: number; + resolvedCount: number; + confidenceUpCount: number; + confidenceDownCount: number; + policyImpactCount: number; + dampedCount: number; + evidenceChangedCount: number; +} + +// Gating statistics +export interface GatingStatistics { + originalEdgeCount: number; + deduplicatedEdgeCount: number; + edgeReductionPercent: number; + totalVerdictCount: number; + surfacedVerdictCount: number; + dampedVerdictCount: number; + duration: string; +} +``` + +### NG-FE-003: API Client + +Create `src/app/core/api/noise-gating.client.ts`: + +```typescript +@Injectable({ providedIn: 'root' }) +export class NoiseGatingApiClient { + // Follow VexHubApiHttpClient patterns + // Signal-based state management + // Caching with Map +} +``` + +### NG-FE-004: Summary Strip Component + +Extend `DeltaSummaryStripComponent` pattern for noise-gating sections: +- New (green), Resolved (blue), ConfidenceUp (teal), ConfidenceDown (orange) +- PolicyImpact (red), Damped (gray), EvidenceChanged (purple) + +### NG-FE-005: Delta Entry Card + +Create `delta-entry-card.component.ts`: +- Display CVE ID, package, status transition +- Confidence change visualization (before -> after with delta %) +- Section-specific styling +- Link to GatingExplainerComponent for details + +### NG-FE-006: Container Component + +Create `noise-gating-delta-report.component.ts`: +- Uses `TabsComponent` with section tabs (badge counts) +- Uses `NoiseGatingSummaryStripComponent` for overview +- Filterable entry list within each tab +- Follows three-pane pattern from compare feature + +### NG-FE-007: Statistics Card + +Create `gating-statistics-card.component.ts`: +- Edge reduction percentage visualization +- Verdict surfacing/damping ratios +- Processing duration display +- Follows `StatCardComponent` patterns + +### NG-FE-008: Triage Integration + +Add to vuln-explorer: +- "Delta Report" tab or drawer +- Trigger from snapshot comparison +- Link from finding detail to delta context + +### NG-FE-009: Module Exports + +Update feature module: +- Export new components +- Add to routing if needed +- Register API client + +## Decisions & Risks + +| Decision | Rationale | +|----------|-----------| +| Compose existing components | ~70% code reuse, consistent UX | +| Signal-based state | Matches existing Angular 17 patterns | +| Section tabs vs flat list | Better UX for categorized changes | +| Lazy-load delta data | Large reports should not block initial render | + +## Execution Log + +| Date | Action | Notes | +|------|--------|-------| +| 2026-01-04 | Sprint created | Based on backend noise-gating completion | +| 2026-01-04 | NG-FE-001 | Added endpoints to VexLensEndpointExtensions.cs, created NoiseGatingApiModels.cs | +| 2026-01-04 | NG-FE-001 | Created ISnapshotStore, IGatingStatisticsStore with in-memory implementations | +| 2026-01-04 | NG-FE-001 | Updated INoiseGate.DiffAsync to accept DeltaReportOptions | +| 2026-01-04 | NG-FE-001 | Registered storage services in VexLensServiceCollectionExtensions | +| 2026-01-04 | NG-FE-002 | Created noise-gating.models.ts with all API types and helper functions | +| 2026-01-04 | NG-FE-003 | Created noise-gating.client.ts with signal-based state and caching | +| 2026-01-04 | NG-FE-004 | Created NoiseGatingSummaryStripComponent with section badges | +| 2026-01-04 | NG-FE-005 | Created DeltaEntryCardComponent for individual entries | +| 2026-01-04 | NG-FE-006 | Created NoiseGatingDeltaReportComponent container with tabs | +| 2026-01-04 | NG-FE-007 | Created GatingStatisticsCardComponent with progress bars | +| 2026-01-04 | NG-FE-009 | Created index.ts barrel export for noise-gating components | +| 2026-01-04 | NG-FE-008 | Integrated noise-gating into TriageCanvasComponent with Delta tab | +| 2026-01-04 | NG-FE-008 | Added keyboard shortcut 'd' for delta tab | +| 2026-01-04 | NG-FE-008 | Updated triage components index.ts to export noise-gating components | +| 2026-01-04 | Sprint complete | All 9 tasks completed | + diff --git a/docs/key-features.md b/docs/key-features.md index a23a87954..d47fe9885 100644 --- a/docs/key-features.md +++ b/docs/key-features.md @@ -2,6 +2,8 @@ > **Core Thesis:** Stella Ops isn't a scanner that outputs findings. It's a platform that outputs **attestable decisions that can be replayed**. That difference survives auditors, regulators, and supply-chain propagation. +> **Looking for the complete feature catalog?** See [`full-features-list.md`](full-features-list.md) for the comprehensive list of all platform capabilities, or [`04_FEATURE_MATRIX.md`](04_FEATURE_MATRIX.md) for tier-by-tier availability. + --- ## At a Glance diff --git a/docs/modules/scanner/operations/secret-leak-detection.md b/docs/modules/scanner/operations/secret-leak-detection.md index 5bd675a0f..2c210293e 100644 --- a/docs/modules/scanner/operations/secret-leak-detection.md +++ b/docs/modules/scanner/operations/secret-leak-detection.md @@ -1,9 +1,25 @@ # Secret Leak Detection (Scanner Operations) -> **Status:** Preview (Sprint 132). Requires `SCANNER-ENG-0007`/`POLICY-READINESS-0001` release bundle and the experimental flag `secret-leak-detection`. +> **Status:** PLANNED - Implementation in progress. See implementation sprints below. +> +> **Previous status:** Preview (Sprint 132). Requires `SCANNER-ENG-0007`/`POLICY-READINESS-0001` release bundle and the experimental flag `secret-leak-detection`. > > **Audience:** Scanner operators, Security Guild, Docs Guild, Offline Kit maintainers. +## Implementation Status + +| Component | Status | Sprint | +|-----------|--------|--------| +| `StellaOps.Scanner.Analyzers.Secrets` plugin | NOT IMPLEMENTED | [SPRINT_20260104_002](../../../implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md) | +| Rule bundle infrastructure | NOT IMPLEMENTED | [SPRINT_20260104_003](../../../implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md) | +| Policy DSL predicates (`secret.*`) | NOT IMPLEMENTED | [SPRINT_20260104_004](../../../implplan/SPRINT_20260104_004_POLICY_secret_dsl_integration.md) | +| Offline Kit integration | NOT IMPLEMENTED | [SPRINT_20260104_005](../../../implplan/SPRINT_20260104_005_AIRGAP_secret_offline_kit.md) | +| Surface.Secrets (credential delivery) | IMPLEMENTED | N/A (already complete) | + +**Note:** The remainder of this document describes the TARGET SPECIFICATION for secret leak detection. The feature is not yet available. Surface.Secrets (operational credential management) is fully implemented and separate from secret leak detection. + +--- + ## 1. Scope & goals - Introduce the **`StellaOps.Scanner.Analyzers.Secrets`** plug-in, which executes deterministic rule bundles against layer content during scans. @@ -29,29 +45,107 @@ Rule bundles ship in the Export Center / Offline Kit under `offline/rules/secret | --- | --- | --- | | `secrets.ruleset.manifest.json` | Lists rule IDs, versions, severity defaults, and hash digests. | Consume during policy drift audits. | | `secrets.ruleset.rules.jsonl` | Newline-delimited definitions (regex/entropy metadata, masking hints). | Loaded by the analyzer at startup. | -| `secrets.ruleset.dsse.json` | DSSE envelope (Signer certificate chain + Attestor proof). | Verify before distributing bundles. | +| `secrets.ruleset.dsse.json` | DSSE envelope (HMAC-SHA256 signature). | Verify before distributing bundles. | -Verification checklist (`stella excititor verify` talks to the configured Attestor service): +### 3.1 Creating custom bundles +Organizations can create custom rule bundles with additional detection patterns: + +```bash +# Create a bundle from rule definition files +stella secrets bundle create \ + --output ./bundles/custom-2026.01 \ + --bundle-id custom.secrets.ruleset \ + --version 2026.01 \ + --rules ./custom-rules/*.json + +# Create and sign in one step +stella secrets bundle create \ + --output ./bundles/custom-2026.01 \ + --bundle-id custom.secrets.ruleset \ + --version 2026.01 \ + --rules ./custom-rules/*.json \ + --sign \ + --key-id my-org-secrets-signer \ + --shared-secret-file /path/to/signing.key ``` + +Rule definition files must follow the JSON schema: + +```json +{ + "id": "myorg.secrets.internal-api-key", + "version": "1.0.0", + "name": "Internal API Key", + "description": "Detects internal API keys with MYORG prefix", + "type": "regex", + "pattern": "MYORG_[A-Z0-9]{32}", + "severity": "high", + "confidence": "high", + "keywords": ["MYORG_"], + "filePatterns": ["*.env", "*.yaml", "*.json"], + "enabled": true +} +``` + +**Rule ID requirements:** +- Must be namespaced (e.g., `myorg.secrets.rule-name`) +- Must start with lowercase letter +- May contain lowercase letters, digits, dots, and hyphens + +### 3.2 Verifying bundles + +Verify bundle integrity and signature before deployment: + +```bash +# Basic verification (checks SHA-256 integrity) +stella secrets bundle verify \ + --bundle ./bundles/2026.01 + +# Full verification with signature check +stella secrets bundle verify \ + --bundle ./bundles/2026.01 \ + --shared-secret-file /path/to/signing.key + +# Verify with trusted key list +stella secrets bundle verify \ + --bundle ./bundles/2026.01 \ + --shared-secret-file /path/to/signing.key \ + --trusted-key-ids stellaops-secrets-signer,my-org-signer +``` + +For air-gapped environments, use the local verification mode (no network calls): + +```bash +stella secrets bundle verify \ + --bundle ./bundles/2026.01 \ + --shared-secret-file /path/to/signing.key \ + --skip-rekor +``` + +Alternatively, use `stella excititor verify` for Attestor-based verification: + +```bash stella excititor verify \ - --attestation offline/rules/secrets/2025.11/secrets.ruleset.dsse.json \ - --digest $(sha256sum offline/rules/secrets/2025.11/secrets.ruleset.rules.jsonl | cut -d' ' -f1) + --attestation offline/rules/secrets/2026.01/secrets.ruleset.dsse.json \ + --digest $(sha256sum offline/rules/secrets/2026.01/secrets.ruleset.rules.jsonl | cut -d' ' -f1) ``` -For air-gapped environments point the CLI at the Offline Kit Attestor mirror (for example `STELLA_ATTESTOR_URL=http://attestor.offline.local`) before running the command. The Attestor instance validates the DSSE envelope against the mirrored Rekor log and embedded certificate chain; no public network access is required. +### 3.3 Deploying bundles -Once verified, copy the manifest + rules to the worker: +Once verified, copy the bundle to the worker: ``` /opt/stellaops/plugins/scanner/analyzers/secrets/ - ├── secrets.ruleset.manifest.json - ├── secrets.ruleset.rules.jsonl - └── secrets.ruleset.dsse.json + |- secrets.ruleset.manifest.json + |- secrets.ruleset.rules.jsonl + |- secrets.ruleset.dsse.json ``` Restart the worker so the analyzer reloads the updated bundle. Bundles are immutable; upgrading requires replacing all three files and restarting. +See [secrets-bundle-rotation.md](./secrets-bundle-rotation.md) for rotation procedures and rollback instructions. + ## 4. Enabling the analyzer 1. **Toggle the feature flag** (WebService + Worker): @@ -161,6 +255,52 @@ rule low_confidence_warn priority 20 { | No findings despite seeded secrets | Ensure bundle hash matches manifest. Run worker with `--secrets-trace` (debug build) to log matched rules locally. | | Policy marks findings as unknown | Upgrade tenant policies to include `secret.*` helpers; older policies silently drop the namespace. | | Air-gapped verification fails | Ensure `STELLA_ATTESTOR_URL` points to the Offline Kit Attestor mirror and rerun `stella excititor verify --attestation --digest `. | +| Signature verification failed | Check shared secret matches signing key. Verify DSSE envelope exists and is not corrupted. See signature troubleshooting below. | +| Bundle integrity check failed | Rules file was modified after signing. Re-download bundle or rebuild from sources. | +| Key not in trusted list | Add signer key ID to `--trusted-key-ids` or update `scanner.secrets.trustedKeyIds` configuration. | + +### 7.1 Signature verification troubleshooting + +**"Signature verification failed" error:** + +1. Verify the shared secret is correct: + ```bash + # Check secret file exists and is readable + cat /path/to/signing.key | wc -c + # Should output the key length (typically 32-64 bytes for base64-encoded keys) + ``` + +2. Check the DSSE envelope format: + ```bash + # Verify envelope is valid JSON + jq . ./bundles/2026.01/secrets.ruleset.dsse.json + ``` + +3. Confirm manifest matches envelope payload: + ```bash + # The envelope payload (base64url-decoded) should match the manifest content + # excluding the signatures field + ``` + +**"Rules file integrity check failed" error:** + +1. Recompute the SHA-256 hash: + ```bash + sha256sum ./bundles/2026.01/secrets.ruleset.rules.jsonl + ``` + +2. Compare with manifest: + ```bash + jq -r '.integrity.rulesSha256' ./bundles/2026.01/secrets.ruleset.manifest.json + ``` + +3. If hashes differ, the rules file was modified. Re-download or rebuild the bundle. + +**"Bundle is not signed" error:** + +The bundle was created without the `--sign` flag. Either: +- Rebuild with signing: `stella secrets bundle create ... --sign --key-id ` +- Skip signature verification: `--skip-signature-verification` (not recommended for production) ## 8. References diff --git a/docs/modules/scanner/operations/secrets-bundle-rotation.md b/docs/modules/scanner/operations/secrets-bundle-rotation.md new file mode 100644 index 000000000..8750547fb --- /dev/null +++ b/docs/modules/scanner/operations/secrets-bundle-rotation.md @@ -0,0 +1,298 @@ +# Secret Detection Bundle Rotation + +> **Audience:** Scanner operators, Security Guild, Offline Kit maintainers. +> +> **Related:** [secret-leak-detection.md](./secret-leak-detection.md) + +## 1. Overview + +Secret detection rule bundles are versioned, immutable artifacts that define the patterns used to detect leaked credentials. This document covers the versioning strategy, rotation procedures, and rollback instructions. + +## 2. Versioning strategy + +Bundles follow CalVer (Calendar Versioning) with the format `YYYY.MM`: + +| Version | Release Type | Notes | +|---------|--------------|-------| +| `2026.01` | Monthly release | Standard monthly update | +| `2026.01.1` | Patch release | Critical rule fix within the month | +| `2026.02` | Monthly release | Next scheduled release | + +**Version precedence:** +- `2026.02` > `2026.01.1` > `2026.01` +- Patch versions (`YYYY.MM.N`) are only used for critical fixes +- Monthly releases reset the patch counter + +**Custom bundles:** +Organizations creating custom bundles should use a prefix to avoid conflicts: +- `myorg.2026.01` for organization-specific bundles +- Or semantic versioning: `1.0.0`, `1.1.0`, etc. + +## 3. Release cadence + +| Release Type | Frequency | Notification | +|--------------|-----------|--------------| +| Monthly release | First week of each month | Release notes, changelog | +| Patch release | As needed for critical rules | Security advisory | +| Breaking changes | Major version bump | Migration guide | + +## 4. Rotation procedures + +### 4.1 Downloading the new bundle + +```bash +# From the Export Center or Offline Kit +curl -O https://export.stellaops.io/rules/secrets/2026.02/secrets.ruleset.manifest.json +curl -O https://export.stellaops.io/rules/secrets/2026.02/secrets.ruleset.rules.jsonl +curl -O https://export.stellaops.io/rules/secrets/2026.02/secrets.ruleset.dsse.json +``` + +For air-gapped environments, obtain the bundle from the Offline Kit media. + +### 4.2 Verifying the bundle + +Always verify bundles before deployment: + +```bash +stella secrets bundle verify \ + --bundle ./2026.02 \ + --shared-secret-file /etc/stellaops/secrets-signing.key \ + --trusted-key-ids stellaops-secrets-signer +``` + +Expected output: +``` +Bundle verified successfully. + Bundle ID: secrets.ruleset + Version: 2026.02 + Rule count: 18 + Enabled rules: 18 + Signed by: stellaops-secrets-signer + Signed at: 2026-02-01T00:00:00Z +``` + +### 4.3 Staged rollout + +For production environments, use a staged rollout: + +**Stage 1: Canary (1 worker)** +```bash +# Deploy to canary worker +scp -r ./2026.02/* canary-worker:/opt/stellaops/plugins/scanner/analyzers/secrets/ +ssh canary-worker 'systemctl restart stellaops-scanner-worker' + +# Monitor for 24 hours +# Check logs, metrics, and finding counts +``` + +**Stage 2: Ring 1 (10% of workers)** +```bash +# Deploy to ring 1 workers +ansible-playbook -l ring1 deploy-secrets-bundle.yml -e bundle_version=2026.02 +``` + +**Stage 3: Full rollout (all workers)** +```bash +# Deploy to all workers +ansible-playbook deploy-secrets-bundle.yml -e bundle_version=2026.02 +``` + +### 4.4 Atomic deployment + +For single-worker deployments or when downtime is acceptable: + +```bash +# Stop the worker +systemctl stop stellaops-scanner-worker + +# Backup current bundle +cp -r /opt/stellaops/plugins/scanner/analyzers/secrets{,.backup} + +# Deploy new bundle +cp -r ./2026.02/* /opt/stellaops/plugins/scanner/analyzers/secrets/ + +# Start the worker +systemctl start stellaops-scanner-worker + +# Verify startup +journalctl -u stellaops-scanner-worker | grep SecretsAnalyzerHost +``` + +### 4.5 Using symlinks (recommended) + +For zero-downtime rotations, use the symlink pattern: + +```bash +# Directory structure +/opt/stellaops/plugins/scanner/analyzers/secrets/ + bundles/ + 2026.01/ + secrets.ruleset.manifest.json + secrets.ruleset.rules.jsonl + secrets.ruleset.dsse.json + 2026.02/ + secrets.ruleset.manifest.json + secrets.ruleset.rules.jsonl + secrets.ruleset.dsse.json + current -> bundles/2026.02 # Symlink +``` + +Rotation with symlinks: +```bash +# Deploy new bundle (no restart needed yet) +cp -r ./2026.02 /opt/stellaops/plugins/scanner/analyzers/secrets/bundles/ + +# Atomic switch +ln -sfn bundles/2026.02 /opt/stellaops/plugins/scanner/analyzers/secrets/current + +# Restart worker to pick up new bundle +systemctl restart stellaops-scanner-worker +``` + +## 5. Rollback procedures + +### 5.1 Quick rollback + +If issues are detected after deployment: + +```bash +# With symlinks (fastest) +ln -sfn bundles/2026.01 /opt/stellaops/plugins/scanner/analyzers/secrets/current +systemctl restart stellaops-scanner-worker + +# Without symlinks +cp -r /opt/stellaops/plugins/scanner/analyzers/secrets.backup/* \ + /opt/stellaops/plugins/scanner/analyzers/secrets/ +systemctl restart stellaops-scanner-worker +``` + +### 5.2 Identifying rollback triggers + +Roll back immediately if you observe: + +| Symptom | Likely Cause | Action | +|---------|--------------|--------| +| Worker fails to start | Bundle corruption or invalid rules | Rollback + investigate | +| Finding count drops to zero | All rules disabled or regex errors | Rollback + check manifest | +| Finding count spikes 10x+ | Overly broad new patterns | Rollback + review rules | +| High CPU usage | Catastrophic regex backtracking | Rollback + report to Security Guild | +| Signature verification failures | Key mismatch or tampering | Rollback + verify bundle source | + +### 5.3 Post-rollback verification + +After rolling back: + +```bash +# Verify worker is healthy +systemctl status stellaops-scanner-worker + +# Check bundle version in logs +journalctl -u stellaops-scanner-worker | grep "Loaded bundle" + +# Verify finding generation (run a test scan) +stella scan --target test-image:latest --secrets-only +``` + +## 6. Bundle retention + +Retain previous bundle versions for rollback capability: + +| Environment | Retention | +|-------------|-----------| +| Production | Last 3 versions | +| Staging | Last 2 versions | +| Development | Latest only | + +Cleanup script: +```bash +#!/bin/bash +BUNDLE_DIR=/opt/stellaops/plugins/scanner/analyzers/secrets/bundles +KEEP=3 + +ls -dt ${BUNDLE_DIR}/*/ | tail -n +$((KEEP+1)) | xargs rm -rf +``` + +## 7. Monitoring rotation + +Key metrics to monitor during rotation: + +| Metric | Baseline | Alert Threshold | +|--------|----------|-----------------| +| `scanner.secret.finding_total` | Varies | +/- 50% from baseline | +| `scanner.secret.scan_duration_ms` | < 100ms | > 500ms | +| `scanner.secret.bundle_load_errors` | 0 | > 0 | +| Worker restart success | 100% | < 100% | + +Prometheus alert example: +```yaml +- alert: SecretBundleRotationAnomaly + expr: | + abs( + sum(rate(scanner_secret_finding_total[5m])) + - sum(rate(scanner_secret_finding_total[5m] offset 1h)) + ) / sum(rate(scanner_secret_finding_total[5m] offset 1h)) > 0.5 + for: 15m + labels: + severity: warning + annotations: + summary: "Secret finding rate changed significantly after bundle rotation" +``` + +## 8. Air-gapped rotation + +For air-gapped environments: + +1. **Obtain bundle from secure media:** + ```bash + # Mount offline kit media + mount /dev/sr0 /mnt/offline-kit + + # Copy bundle + cp -r /mnt/offline-kit/rules/secrets/2026.02 \ + /opt/stellaops/plugins/scanner/analyzers/secrets/bundles/ + ``` + +2. **Verify with local secret:** + ```bash + stella secrets bundle verify \ + --bundle /opt/stellaops/plugins/scanner/analyzers/secrets/bundles/2026.02 \ + --shared-secret-file /etc/stellaops/offline-signing.key \ + --skip-rekor + ``` + +3. **Follow standard rotation procedure (Section 4).** + +## 9. Emergency procedures + +### 9.1 Disabling secret detection + +If secret detection must be disabled entirely: + +```bash +# Disable via configuration +echo 'scanner.features.experimental.secret-leak-detection: false' >> /etc/stellaops/scanner.yaml + +# Restart worker +systemctl restart stellaops-scanner-worker +``` + +### 9.2 Emergency rule disable + +To disable a specific problematic rule without full rotation: + +1. Edit the manifest to set `enabled: false` for the rule +2. This breaks signature verification (expected) +3. Configure worker to skip signature verification temporarily: + ```yaml + scanner: + secrets: + skipSignatureVerification: true # TEMPORARY - re-enable after fix + ``` +4. Restart worker +5. Request emergency patch release from Security Guild + +## 10. References + +- [secret-leak-detection.md](./secret-leak-detection.md) - Main secret detection documentation +- [SPRINT_20260104_003_SCANNER_secret_rule_bundles.md](../../../implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md) - Implementation sprint +- [dsse-rekor-operator-guide.md](./dsse-rekor-operator-guide.md) - DSSE and Rekor verification diff --git a/offline/rules/secrets/sources/aws-access-key.json b/offline/rules/secrets/sources/aws-access-key.json new file mode 100644 index 000000000..157006315 --- /dev/null +++ b/offline/rules/secrets/sources/aws-access-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.aws-access-key", + "version": "1.0.0", + "name": "AWS Access Key ID", + "description": "Detects AWS Access Key IDs which start with AKIA, ASIA, AIDA, AGPA, AROA, AIPA, ANPA, or ANVA", + "type": "regex", + "pattern": "(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}", + "severity": "high", + "confidence": "high", + "keywords": ["AKIA", "ASIA", "AIDA", "AGPA", "AROA", "AIPA", "ANPA", "ANVA", "aws"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.tf", "*.tfvars", "*.config"], + "enabled": true, + "tags": ["aws", "cloud", "credentials"], + "references": [ + "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html" + ] +} diff --git a/offline/rules/secrets/sources/aws-secret-key.json b/offline/rules/secrets/sources/aws-secret-key.json new file mode 100644 index 000000000..74a8890a8 --- /dev/null +++ b/offline/rules/secrets/sources/aws-secret-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.aws-secret-key", + "version": "1.0.0", + "name": "AWS Secret Access Key", + "description": "Detects AWS Secret Access Keys (40-character base64 strings near AWS context)", + "type": "regex", + "pattern": "(?i)(?:aws[_-]?secret[_-]?(?:access[_-]?)?key|secret[_-]?key)['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9/+=]{40})['\"]?", + "severity": "critical", + "confidence": "high", + "keywords": ["aws_secret", "secret_key", "secret_access_key", "aws"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.tf", "*.tfvars", "*.config", "*.sh", "*.bash"], + "enabled": true, + "tags": ["aws", "cloud", "credentials"], + "references": [ + "https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html" + ] +} diff --git a/offline/rules/secrets/sources/azure-storage-key.json b/offline/rules/secrets/sources/azure-storage-key.json new file mode 100644 index 000000000..bf61f7b47 --- /dev/null +++ b/offline/rules/secrets/sources/azure-storage-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.azure-storage-key", + "version": "1.0.0", + "name": "Azure Storage Account Key", + "description": "Detects Azure Storage Account access keys (base64 encoded, 88 chars)", + "type": "regex", + "pattern": "(?i)(?:AccountKey|azure[_-]?storage[_-]?key)['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9+/]{86}==)['\"]?", + "severity": "critical", + "confidence": "high", + "keywords": ["AccountKey", "azure_storage", "DefaultEndpointsProtocol"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "*.tf", "*.tfvars", "appsettings.json", "web.config"], + "enabled": true, + "tags": ["azure", "cloud", "credentials", "storage"], + "references": [ + "https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage" + ] +} diff --git a/offline/rules/secrets/sources/database-connection-string.json b/offline/rules/secrets/sources/database-connection-string.json new file mode 100644 index 000000000..ddc78a812 --- /dev/null +++ b/offline/rules/secrets/sources/database-connection-string.json @@ -0,0 +1,16 @@ +{ + "id": "stellaops.secrets.database-connection-string", + "version": "1.0.0", + "name": "Database Connection String with Credentials", + "description": "Detects database connection strings containing embedded credentials", + "type": "regex", + "pattern": "(?i)(?:postgres|mysql|mongodb|sqlserver|mssql)://[^:]+:[^@]+@[^/]+", + "severity": "critical", + "confidence": "high", + "keywords": ["postgres://", "mysql://", "mongodb://", "sqlserver://", "connection"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "appsettings.json", "*.xml"], + "enabled": true, + "allowlistPatterns": ["localhost", "127\\.0\\.0\\.1", "\\$\\{", "\\{\\{"], + "tags": ["database", "credentials", "connection-string"], + "references": [] +} diff --git a/offline/rules/secrets/sources/datadog-api-key.json b/offline/rules/secrets/sources/datadog-api-key.json new file mode 100644 index 000000000..0a49e75d0 --- /dev/null +++ b/offline/rules/secrets/sources/datadog-api-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.datadog-api-key", + "version": "1.0.0", + "name": "Datadog API Key", + "description": "Detects Datadog API keys", + "type": "regex", + "pattern": "(?i)(?:datadog[_-]?api[_-]?key|DD_API_KEY)['\"]?\\s*[:=]\\s*['\"]?([a-f0-9]{32})['\"]?", + "severity": "high", + "confidence": "medium", + "keywords": ["DD_API_KEY", "datadog_api_key", "datadog"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config"], + "enabled": true, + "tags": ["datadog", "monitoring", "observability", "credentials", "api-key"], + "references": [ + "https://docs.datadoghq.com/account_management/api-app-keys/" + ] +} diff --git a/offline/rules/secrets/sources/discord-bot-token.json b/offline/rules/secrets/sources/discord-bot-token.json new file mode 100644 index 000000000..59715b5b5 --- /dev/null +++ b/offline/rules/secrets/sources/discord-bot-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.discord-bot-token", + "version": "1.0.0", + "name": "Discord Bot Token", + "description": "Detects Discord bot tokens", + "type": "regex", + "pattern": "[MN][A-Za-z\\d]{23,}\\.[\\w-]{6}\\.[\\w-]{27,}", + "severity": "high", + "confidence": "high", + "keywords": ["discord", "bot", "token"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "*.py", "*.js"], + "enabled": true, + "tags": ["discord", "messaging", "bot", "credentials", "token"], + "references": [ + "https://discord.com/developers/docs/topics/oauth2" + ] +} diff --git a/offline/rules/secrets/sources/docker-hub-token.json b/offline/rules/secrets/sources/docker-hub-token.json new file mode 100644 index 000000000..00fcedc65 --- /dev/null +++ b/offline/rules/secrets/sources/docker-hub-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.docker-hub-token", + "version": "1.0.0", + "name": "Docker Hub Access Token", + "description": "Detects Docker Hub personal access tokens", + "type": "regex", + "pattern": "dckr_pat_[A-Za-z0-9_-]{27}", + "severity": "high", + "confidence": "high", + "keywords": ["dckr_pat_", "docker", "registry"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.sh", ".docker/config.json"], + "enabled": true, + "tags": ["docker", "container", "registry", "credentials", "token"], + "references": [ + "https://docs.docker.com/docker-hub/access-tokens/" + ] +} diff --git a/offline/rules/secrets/sources/gcp-service-account.json b/offline/rules/secrets/sources/gcp-service-account.json new file mode 100644 index 000000000..722f17f1e --- /dev/null +++ b/offline/rules/secrets/sources/gcp-service-account.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.gcp-service-account", + "version": "1.0.0", + "name": "GCP Service Account Key", + "description": "Detects GCP service account JSON key files by their structure", + "type": "regex", + "pattern": "\"type\"\\s*:\\s*\"service_account\"[\\s\\S]{0,500}\"private_key\"\\s*:\\s*\"-----BEGIN", + "severity": "critical", + "confidence": "high", + "keywords": ["service_account", "private_key", "gcp", "google", "client_email"], + "filePatterns": ["*.json"], + "enabled": true, + "tags": ["gcp", "google", "cloud", "credentials", "service-account"], + "references": [ + "https://cloud.google.com/iam/docs/keys-create-delete" + ] +} diff --git a/offline/rules/secrets/sources/generic-api-key.json b/offline/rules/secrets/sources/generic-api-key.json new file mode 100644 index 000000000..a01850880 --- /dev/null +++ b/offline/rules/secrets/sources/generic-api-key.json @@ -0,0 +1,15 @@ +{ + "id": "stellaops.secrets.generic-api-key", + "version": "1.0.0", + "name": "Generic API Key", + "description": "Detects generic API key patterns in configuration", + "type": "regex", + "pattern": "(?i)(?:api[_-]?key|apikey|api[_-]?secret)['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9_-]{20,})['\"]?", + "severity": "medium", + "confidence": "low", + "keywords": ["api_key", "apikey", "api-key", "api_secret"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config"], + "enabled": true, + "tags": ["api-key", "credentials", "generic"], + "references": [] +} diff --git a/offline/rules/secrets/sources/generic-password.json b/offline/rules/secrets/sources/generic-password.json new file mode 100644 index 000000000..535b7a746 --- /dev/null +++ b/offline/rules/secrets/sources/generic-password.json @@ -0,0 +1,16 @@ +{ + "id": "stellaops.secrets.generic-password", + "version": "1.0.0", + "name": "Generic Password Assignment", + "description": "Detects hardcoded password assignments in configuration and code", + "type": "regex", + "pattern": "(?i)(?:password|passwd|pwd)['\"]?\\s*[:=]\\s*['\"]([^'\"\\s]{8,})['\"]", + "severity": "high", + "confidence": "low", + "keywords": ["password", "passwd", "pwd"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "*.xml"], + "enabled": true, + "allowlistPatterns": ["\\$\\{", "\\{\\{", "%[A-Z_]+%", "\\$env:", "process\\.env"], + "tags": ["password", "credentials", "generic"], + "references": [] +} diff --git a/offline/rules/secrets/sources/github-app-token.json b/offline/rules/secrets/sources/github-app-token.json new file mode 100644 index 000000000..6a6c3e7a8 --- /dev/null +++ b/offline/rules/secrets/sources/github-app-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.github-app-token", + "version": "1.0.0", + "name": "GitHub App Installation Token", + "description": "Detects GitHub App installation access tokens", + "type": "regex", + "pattern": "ghs_[A-Za-z0-9_]{36,255}", + "severity": "high", + "confidence": "high", + "keywords": ["ghs_", "github", "app"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.sh", "*.bash"], + "enabled": true, + "tags": ["github", "vcs", "credentials", "token", "app"], + "references": [ + "https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-an-installation-access-token-for-a-github-app" + ] +} diff --git a/offline/rules/secrets/sources/github-pat.json b/offline/rules/secrets/sources/github-pat.json new file mode 100644 index 000000000..20a826aa4 --- /dev/null +++ b/offline/rules/secrets/sources/github-pat.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.github-pat", + "version": "1.0.0", + "name": "GitHub Personal Access Token", + "description": "Detects GitHub Personal Access Tokens (classic and fine-grained)", + "type": "regex", + "pattern": "(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,255}", + "severity": "critical", + "confidence": "high", + "keywords": ["ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.sh", "*.bash", "*.md", "*.txt"], + "enabled": true, + "tags": ["github", "vcs", "credentials", "token"], + "references": [ + "https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens" + ] +} diff --git a/offline/rules/secrets/sources/gitlab-pat.json b/offline/rules/secrets/sources/gitlab-pat.json new file mode 100644 index 000000000..0a93f982e --- /dev/null +++ b/offline/rules/secrets/sources/gitlab-pat.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.gitlab-pat", + "version": "1.0.0", + "name": "GitLab Personal Access Token", + "description": "Detects GitLab Personal Access Tokens (glpat- prefix)", + "type": "regex", + "pattern": "glpat-[A-Za-z0-9_-]{20,}", + "severity": "critical", + "confidence": "high", + "keywords": ["glpat-", "gitlab"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.sh", "*.bash", ".gitlab-ci.yml"], + "enabled": true, + "tags": ["gitlab", "vcs", "credentials", "token"], + "references": [ + "https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html" + ] +} diff --git a/offline/rules/secrets/sources/heroku-api-key.json b/offline/rules/secrets/sources/heroku-api-key.json new file mode 100644 index 000000000..17efb2cfd --- /dev/null +++ b/offline/rules/secrets/sources/heroku-api-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.heroku-api-key", + "version": "1.0.0", + "name": "Heroku API Key", + "description": "Detects Heroku API keys", + "type": "regex", + "pattern": "(?i)(?:heroku[_-]?api[_-]?key|HEROKU_API_KEY)['\"]?\\s*[:=]\\s*['\"]?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})['\"]?", + "severity": "high", + "confidence": "high", + "keywords": ["HEROKU_API_KEY", "heroku_api_key", "heroku"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.sh", "Procfile"], + "enabled": true, + "tags": ["heroku", "paas", "credentials", "api-key"], + "references": [ + "https://devcenter.heroku.com/articles/platform-api-quickstart" + ] +} diff --git a/offline/rules/secrets/sources/jwt-secret.json b/offline/rules/secrets/sources/jwt-secret.json new file mode 100644 index 000000000..9b6f8ef2d --- /dev/null +++ b/offline/rules/secrets/sources/jwt-secret.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.jwt-secret", + "version": "1.0.0", + "name": "JWT Secret Key", + "description": "Detects JWT secret keys in configuration", + "type": "regex", + "pattern": "(?i)(?:jwt[_-]?secret|jwt[_-]?key|secret[_-]?key)['\"]?\\s*[:=]\\s*['\"]?([A-Za-z0-9+/=_-]{32,})['\"]?", + "severity": "high", + "confidence": "medium", + "keywords": ["jwt_secret", "jwt_key", "secret_key", "JWT"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "appsettings.json"], + "enabled": true, + "tags": ["jwt", "authentication", "credentials"], + "references": [ + "https://jwt.io/introduction" + ] +} diff --git a/offline/rules/secrets/sources/mailchimp-api-key.json b/offline/rules/secrets/sources/mailchimp-api-key.json new file mode 100644 index 000000000..d8686a4d5 --- /dev/null +++ b/offline/rules/secrets/sources/mailchimp-api-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.mailchimp-api-key", + "version": "1.0.0", + "name": "Mailchimp API Key", + "description": "Detects Mailchimp API keys", + "type": "regex", + "pattern": "[a-f0-9]{32}-us[0-9]{1,2}", + "severity": "medium", + "confidence": "high", + "keywords": ["mailchimp", "-us", "api_key"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config"], + "enabled": true, + "tags": ["mailchimp", "email", "marketing", "credentials", "api-key"], + "references": [ + "https://mailchimp.com/help/about-api-keys/" + ] +} diff --git a/offline/rules/secrets/sources/npm-token.json b/offline/rules/secrets/sources/npm-token.json new file mode 100644 index 000000000..848401352 --- /dev/null +++ b/offline/rules/secrets/sources/npm-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.npm-token", + "version": "1.0.0", + "name": "NPM Access Token", + "description": "Detects NPM access tokens", + "type": "regex", + "pattern": "npm_[A-Za-z0-9]{36}", + "severity": "high", + "confidence": "high", + "keywords": ["npm_", "npmrc", "_authToken"], + "filePatterns": [".npmrc", "*.yml", "*.yaml", "*.json", "*.env", "*.sh"], + "enabled": true, + "tags": ["npm", "package-manager", "credentials", "token"], + "references": [ + "https://docs.npmjs.com/about-access-tokens" + ] +} diff --git a/offline/rules/secrets/sources/nuget-api-key.json b/offline/rules/secrets/sources/nuget-api-key.json new file mode 100644 index 000000000..ed0c064ac --- /dev/null +++ b/offline/rules/secrets/sources/nuget-api-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.nuget-api-key", + "version": "1.0.0", + "name": "NuGet API Key", + "description": "Detects NuGet.org API keys", + "type": "regex", + "pattern": "oy2[a-z0-9]{43}", + "severity": "high", + "confidence": "high", + "keywords": ["oy2", "nuget", "NuGetApiKey"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.config", "nuget.config", "*.csproj", "*.ps1"], + "enabled": true, + "tags": ["nuget", "dotnet", "package-manager", "credentials", "api-key"], + "references": [ + "https://docs.microsoft.com/en-us/nuget/nuget-org/publish-a-package#create-api-keys" + ] +} diff --git a/offline/rules/secrets/sources/private-key-ec.json b/offline/rules/secrets/sources/private-key-ec.json new file mode 100644 index 000000000..54d435ba4 --- /dev/null +++ b/offline/rules/secrets/sources/private-key-ec.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.private-key-ec", + "version": "1.0.0", + "name": "EC Private Key", + "description": "Detects EC (Elliptic Curve) private keys in PEM format", + "type": "regex", + "pattern": "-----BEGIN EC PRIVATE KEY-----[\\s\\S]{50,}-----END EC PRIVATE KEY-----", + "severity": "critical", + "confidence": "high", + "keywords": ["BEGIN EC PRIVATE KEY", "END EC PRIVATE KEY"], + "filePatterns": ["*.pem", "*.key", "*.txt", "*.env", "*.yml", "*.yaml", "*.json", "*.config"], + "enabled": true, + "tags": ["cryptography", "private-key", "ecdsa"], + "references": [ + "https://www.rfc-editor.org/rfc/rfc5915" + ] +} diff --git a/offline/rules/secrets/sources/private-key-generic.json b/offline/rules/secrets/sources/private-key-generic.json new file mode 100644 index 000000000..18bd005d7 --- /dev/null +++ b/offline/rules/secrets/sources/private-key-generic.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.private-key-generic", + "version": "1.0.0", + "name": "Generic Private Key", + "description": "Detects generic private keys in PEM format (PKCS#8)", + "type": "regex", + "pattern": "-----BEGIN PRIVATE KEY-----[\\s\\S]{100,}-----END PRIVATE KEY-----", + "severity": "critical", + "confidence": "high", + "keywords": ["BEGIN PRIVATE KEY", "END PRIVATE KEY"], + "filePatterns": ["*.pem", "*.key", "*.txt", "*.env", "*.yml", "*.yaml", "*.json", "*.config"], + "enabled": true, + "tags": ["cryptography", "private-key", "pkcs8"], + "references": [ + "https://www.rfc-editor.org/rfc/rfc5958" + ] +} diff --git a/offline/rules/secrets/sources/private-key-openssh.json b/offline/rules/secrets/sources/private-key-openssh.json new file mode 100644 index 000000000..1f30ea1ad --- /dev/null +++ b/offline/rules/secrets/sources/private-key-openssh.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.private-key-openssh", + "version": "1.0.0", + "name": "OpenSSH Private Key", + "description": "Detects OpenSSH private keys (newer format)", + "type": "regex", + "pattern": "-----BEGIN OPENSSH PRIVATE KEY-----[\\s\\S]{50,}-----END OPENSSH PRIVATE KEY-----", + "severity": "critical", + "confidence": "high", + "keywords": ["BEGIN OPENSSH PRIVATE KEY", "END OPENSSH PRIVATE KEY"], + "filePatterns": ["*.pem", "*.key", "id_rsa", "id_ed25519", "id_ecdsa", "*.txt"], + "enabled": true, + "tags": ["cryptography", "private-key", "ssh", "openssh"], + "references": [ + "https://man.openbsd.org/ssh-keygen" + ] +} diff --git a/offline/rules/secrets/sources/private-key-rsa.json b/offline/rules/secrets/sources/private-key-rsa.json new file mode 100644 index 000000000..245cc4750 --- /dev/null +++ b/offline/rules/secrets/sources/private-key-rsa.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.private-key-rsa", + "version": "1.0.0", + "name": "RSA Private Key", + "description": "Detects RSA private keys in PEM format", + "type": "regex", + "pattern": "-----BEGIN RSA PRIVATE KEY-----[\\s\\S]{100,}-----END RSA PRIVATE KEY-----", + "severity": "critical", + "confidence": "high", + "keywords": ["BEGIN RSA PRIVATE KEY", "END RSA PRIVATE KEY"], + "filePatterns": ["*.pem", "*.key", "*.txt", "*.env", "*.yml", "*.yaml", "*.json", "*.config"], + "enabled": true, + "tags": ["cryptography", "private-key", "rsa"], + "references": [ + "https://www.rfc-editor.org/rfc/rfc7468" + ] +} diff --git a/offline/rules/secrets/sources/pypi-token.json b/offline/rules/secrets/sources/pypi-token.json new file mode 100644 index 000000000..7c6164781 --- /dev/null +++ b/offline/rules/secrets/sources/pypi-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.pypi-token", + "version": "1.0.0", + "name": "PyPI API Token", + "description": "Detects PyPI API tokens", + "type": "regex", + "pattern": "pypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}", + "severity": "high", + "confidence": "high", + "keywords": ["pypi-", "pypi.org"], + "filePatterns": [".pypirc", "*.yml", "*.yaml", "*.json", "*.env", "*.sh", "*.toml"], + "enabled": true, + "tags": ["pypi", "python", "package-manager", "credentials", "token"], + "references": [ + "https://pypi.org/help/#apitoken" + ] +} diff --git a/offline/rules/secrets/sources/sendgrid-api-key.json b/offline/rules/secrets/sources/sendgrid-api-key.json new file mode 100644 index 000000000..224ec832c --- /dev/null +++ b/offline/rules/secrets/sources/sendgrid-api-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.sendgrid-api-key", + "version": "1.0.0", + "name": "SendGrid API Key", + "description": "Detects SendGrid API keys", + "type": "regex", + "pattern": "SG\\.[A-Za-z0-9_-]{22}\\.[A-Za-z0-9_-]{43}", + "severity": "high", + "confidence": "high", + "keywords": ["SG.", "sendgrid"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config"], + "enabled": true, + "tags": ["sendgrid", "email", "credentials", "api-key"], + "references": [ + "https://docs.sendgrid.com/ui/account-and-settings/api-keys" + ] +} diff --git a/offline/rules/secrets/sources/slack-token.json b/offline/rules/secrets/sources/slack-token.json new file mode 100644 index 000000000..df1565c3d --- /dev/null +++ b/offline/rules/secrets/sources/slack-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.slack-token", + "version": "1.0.0", + "name": "Slack Token", + "description": "Detects Slack Bot, User, and Webhook tokens", + "type": "regex", + "pattern": "xox[baprs]-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*", + "severity": "high", + "confidence": "high", + "keywords": ["xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-", "slack"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.sh", "*.bash", "*.config"], + "enabled": true, + "tags": ["slack", "messaging", "credentials", "token"], + "references": [ + "https://api.slack.com/authentication/token-types" + ] +} diff --git a/offline/rules/secrets/sources/slack-webhook.json b/offline/rules/secrets/sources/slack-webhook.json new file mode 100644 index 000000000..76ac3b7fd --- /dev/null +++ b/offline/rules/secrets/sources/slack-webhook.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.slack-webhook", + "version": "1.0.0", + "name": "Slack Webhook URL", + "description": "Detects Slack incoming webhook URLs", + "type": "regex", + "pattern": "https://hooks\\.slack\\.com/services/T[A-Z0-9]{8,}/B[A-Z0-9]{8,}/[A-Za-z0-9]{24}", + "severity": "medium", + "confidence": "high", + "keywords": ["hooks.slack.com", "webhook", "slack"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.sh", "*.bash", "*.config", "*.md"], + "enabled": true, + "tags": ["slack", "messaging", "webhook"], + "references": [ + "https://api.slack.com/messaging/webhooks" + ] +} diff --git a/offline/rules/secrets/sources/stripe-restricted-key.json b/offline/rules/secrets/sources/stripe-restricted-key.json new file mode 100644 index 000000000..cd0f22e0f --- /dev/null +++ b/offline/rules/secrets/sources/stripe-restricted-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.stripe-restricted-key", + "version": "1.0.0", + "name": "Stripe Restricted Key", + "description": "Detects Stripe restricted API keys", + "type": "regex", + "pattern": "rk_(?:live|test)_[A-Za-z0-9]{24,}", + "severity": "high", + "confidence": "high", + "keywords": ["rk_live_", "rk_test_", "stripe"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "*.js", "*.ts", "*.py", "*.rb"], + "enabled": true, + "tags": ["stripe", "payment", "credentials", "api-key"], + "references": [ + "https://stripe.com/docs/keys#limit-access" + ] +} diff --git a/offline/rules/secrets/sources/stripe-secret-key.json b/offline/rules/secrets/sources/stripe-secret-key.json new file mode 100644 index 000000000..fdcc0a23b --- /dev/null +++ b/offline/rules/secrets/sources/stripe-secret-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.stripe-secret-key", + "version": "1.0.0", + "name": "Stripe Secret Key", + "description": "Detects Stripe secret API keys (live and test)", + "type": "regex", + "pattern": "sk_(?:live|test)_[A-Za-z0-9]{24,}", + "severity": "critical", + "confidence": "high", + "keywords": ["sk_live_", "sk_test_", "stripe"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "*.js", "*.ts", "*.py", "*.rb"], + "enabled": true, + "tags": ["stripe", "payment", "credentials", "api-key"], + "references": [ + "https://stripe.com/docs/keys" + ] +} diff --git a/offline/rules/secrets/sources/telegram-bot-token.json b/offline/rules/secrets/sources/telegram-bot-token.json new file mode 100644 index 000000000..07a96db5f --- /dev/null +++ b/offline/rules/secrets/sources/telegram-bot-token.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.telegram-bot-token", + "version": "1.0.0", + "name": "Telegram Bot Token", + "description": "Detects Telegram Bot API tokens", + "type": "regex", + "pattern": "[0-9]{8,10}:[A-Za-z0-9_-]{35}", + "severity": "high", + "confidence": "medium", + "keywords": ["telegram", "bot", "api.telegram.org"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config", "*.py", "*.js"], + "enabled": true, + "tags": ["telegram", "messaging", "bot", "credentials", "token"], + "references": [ + "https://core.telegram.org/bots/api#authorizing-your-bot" + ] +} diff --git a/offline/rules/secrets/sources/twilio-api-key.json b/offline/rules/secrets/sources/twilio-api-key.json new file mode 100644 index 000000000..d09fb6e7e --- /dev/null +++ b/offline/rules/secrets/sources/twilio-api-key.json @@ -0,0 +1,17 @@ +{ + "id": "stellaops.secrets.twilio-api-key", + "version": "1.0.0", + "name": "Twilio API Key", + "description": "Detects Twilio API Key SIDs and Auth Tokens", + "type": "regex", + "pattern": "(?:SK[a-f0-9]{32}|AC[a-f0-9]{32})", + "severity": "high", + "confidence": "high", + "keywords": ["SK", "AC", "twilio", "TWILIO"], + "filePatterns": ["*.yml", "*.yaml", "*.json", "*.env", "*.properties", "*.config"], + "enabled": true, + "tags": ["twilio", "messaging", "sms", "credentials", "api-key"], + "references": [ + "https://www.twilio.com/docs/iam/keys/api-key" + ] +} diff --git a/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandGroup.cs b/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandGroup.cs index 70a275e7f..1f6478b4f 100644 --- a/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandGroup.cs +++ b/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandGroup.cs @@ -33,6 +33,9 @@ internal static class BinaryCommandGroup binary.Add(BuildLookupCommand(services, verboseOption, cancellationToken)); binary.Add(BuildFingerprintCommand(services, verboseOption, cancellationToken)); + // Sprint: SPRINT_20260104_001_CLI - Binary call graph digest extraction + binary.Add(BuildCallGraphCommand(services, verboseOption, cancellationToken)); + return binary; } @@ -188,6 +191,70 @@ internal static class BinaryCommandGroup return command; } + // CALLGRAPH-01: stella binary callgraph + private static Command BuildCallGraphCommand( + IServiceProvider services, + Option verboseOption, + CancellationToken cancellationToken) + { + var fileArg = new Argument("file") + { + Description = "Path to binary file to analyze." + }; + + var formatOption = new Option("--format", ["-f"]) + { + Description = "Output format: digest (default), json, summary." + }.SetDefaultValue("digest").FromAmong("digest", "json", "summary"); + + var outputOption = new Option("--output", ["-o"]) + { + Description = "Output file path (default: stdout)." + }; + + var emitSbomOption = new Option("--emit-sbom") + { + Description = "Path to SBOM file to inject callgraph digest as property." + }; + + var scanIdOption = new Option("--scan-id") + { + Description = "Scan ID for graph metadata (default: auto-generated)." + }; + + var command = new Command("callgraph", "Extract call graph and compute deterministic digest.") + { + fileArg, + formatOption, + outputOption, + emitSbomOption, + scanIdOption, + verboseOption + }; + + command.SetAction(parseResult => + { + var file = parseResult.GetValue(fileArg)!; + var format = parseResult.GetValue(formatOption)!; + var output = parseResult.GetValue(outputOption); + var emitSbom = parseResult.GetValue(emitSbomOption); + var scanId = parseResult.GetValue(scanIdOption); + var verbose = parseResult.GetValue(verboseOption); + + return BinaryCommandHandlers.HandleCallGraphAsync( + services, + file, + format, + output, + emitSbom, + scanId, + verbose, + cancellationToken); + }); + + return command; + } + private static Command BuildSubmitCommand( IServiceProvider services, Option verboseOption, diff --git a/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandHandlers.cs b/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandHandlers.cs index a4453df38..7127751f0 100644 --- a/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandHandlers.cs +++ b/src/Cli/StellaOps.Cli/Commands/Binary/BinaryCommandHandlers.cs @@ -5,10 +5,14 @@ // Description: Command handlers for binary reachability operations. // ----------------------------------------------------------------------------- +using System.Globalization; using System.Text.Json; +using System.Text.Json.Nodes; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Spectre.Console; +using StellaOps.Scanner.CallGraph; +using StellaOps.Scanner.CallGraph.Binary; namespace StellaOps.Cli.Commands.Binary; @@ -632,6 +636,238 @@ internal static class BinaryCommandHandlers } } + /// + /// Handle 'stella binary callgraph' command (CALLGRAPH-01). + /// Extracts call graph from native binary and computes deterministic SHA-256 digest. + /// + public static async Task HandleCallGraphAsync( + IServiceProvider services, + string filePath, + string format, + string? outputPath, + string? emitSbomPath, + string? scanId, + bool verbose, + CancellationToken cancellationToken) + { + var loggerFactory = services.GetRequiredService(); + var logger = loggerFactory.CreateLogger("binary-callgraph"); + + try + { + if (!File.Exists(filePath)) + { + AnsiConsole.MarkupLine($"[red]Error:[/] File not found: {filePath}"); + return ExitCodes.FileNotFound; + } + + // Resolve scan ID (auto-generate if not provided) + var effectiveScanId = scanId ?? $"cli-{Path.GetFileName(filePath)}-{DateTime.UtcNow:yyyyMMddHHmmss}"; + + CallGraphSnapshot snapshot = null!; + + await AnsiConsole.Status() + .StartAsync("Extracting binary call graph...", async ctx => + { + ctx.Status("Loading binary..."); + + // Get the binary call graph extractor + var timeProvider = services.GetService() ?? TimeProvider.System; + var extractorLogger = loggerFactory.CreateLogger(); + var extractor = new BinaryCallGraphExtractor(extractorLogger, timeProvider); + + ctx.Status("Analyzing symbols and relocations..."); + + var request = new CallGraphExtractionRequest( + ScanId: effectiveScanId, + Language: "native", + TargetPath: filePath); + + snapshot = await extractor.ExtractAsync(request, cancellationToken); + + ctx.Status("Computing digest..."); + }); + + // Format output based on requested format + string output; + switch (format) + { + case "digest": + output = snapshot.GraphDigest; + break; + + case "json": + output = JsonSerializer.Serialize(new + { + scanId = snapshot.ScanId, + graphDigest = snapshot.GraphDigest, + language = snapshot.Language, + extractedAt = snapshot.ExtractedAt.ToString("O", CultureInfo.InvariantCulture), + nodeCount = snapshot.Nodes.Length, + edgeCount = snapshot.Edges.Length, + entrypointCount = snapshot.EntrypointIds.Length, + nodes = snapshot.Nodes, + edges = snapshot.Edges, + entrypointIds = snapshot.EntrypointIds + }, JsonOptions); + break; + + case "summary": + default: + output = string.Join(Environment.NewLine, + [ + $"GraphDigest: {snapshot.GraphDigest}", + $"ScanId: {snapshot.ScanId}", + $"Language: {snapshot.Language}", + $"ExtractedAt: {snapshot.ExtractedAt:O}", + $"Nodes: {snapshot.Nodes.Length}", + $"Edges: {snapshot.Edges.Length}", + $"Entrypoints: {snapshot.EntrypointIds.Length}" + ]); + break; + } + + // Write output + if (!string.IsNullOrWhiteSpace(outputPath)) + { + await File.WriteAllTextAsync(outputPath, output, cancellationToken); + AnsiConsole.MarkupLine($"[green]Output written to:[/] {outputPath}"); + } + else if (format == "digest") + { + // For digest-only format, just output the digest + Console.WriteLine(output); + } + else + { + AnsiConsole.WriteLine(output); + } + + // Inject into SBOM if requested + if (!string.IsNullOrWhiteSpace(emitSbomPath)) + { + var sbomResult = await InjectCallGraphDigestIntoSbomAsync( + emitSbomPath, + filePath, + snapshot, + cancellationToken); + + if (sbomResult == 0) + { + AnsiConsole.MarkupLine($"[green]Callgraph digest injected into SBOM:[/] {emitSbomPath}"); + } + else + { + AnsiConsole.MarkupLine($"[yellow]Warning:[/] Failed to inject digest into SBOM"); + } + } + + if (verbose) + { + logger.LogInformation( + "Extracted call graph for {Path}: {Nodes} nodes, {Edges} edges, digest={Digest}", + filePath, + snapshot.Nodes.Length, + snapshot.Edges.Length, + snapshot.GraphDigest); + } + + return ExitCodes.Success; + } + catch (NotSupportedException ex) + { + AnsiConsole.MarkupLine($"[red]Error:[/] {ex.Message}"); + logger.LogError(ex, "Unsupported binary format for {Path}", filePath); + return ExitCodes.InvalidArguments; + } + catch (Exception ex) + { + AnsiConsole.MarkupLine($"[red]Error:[/] {ex.Message}"); + logger.LogError(ex, "Failed to extract call graph for {Path}", filePath); + return ExitCodes.GeneralError; + } + } + + /// + /// Injects callgraph digest as a property into a CycloneDX SBOM. + /// + private static async Task InjectCallGraphDigestIntoSbomAsync( + string sbomPath, + string binaryPath, + CallGraphSnapshot snapshot, + CancellationToken cancellationToken) + { + if (!File.Exists(sbomPath)) + { + return 1; + } + + try + { + var sbomJson = await File.ReadAllTextAsync(sbomPath, cancellationToken); + var doc = JsonNode.Parse(sbomJson) as JsonObject; + + if (doc == null) + { + return 1; + } + + // Ensure metadata.properties exists + var metadata = doc["metadata"] as JsonObject; + if (metadata == null) + { + metadata = new JsonObject(); + doc["metadata"] = metadata; + } + + var properties = metadata["properties"] as JsonArray; + if (properties == null) + { + properties = new JsonArray(); + metadata["properties"] = properties; + } + + var binaryName = Path.GetFileName(binaryPath); + + // Add callgraph properties using stellaops namespace + properties.Add(new JsonObject + { + ["name"] = $"stellaops:callgraph:digest:{binaryName}", + ["value"] = snapshot.GraphDigest + }); + properties.Add(new JsonObject + { + ["name"] = $"stellaops:callgraph:nodeCount:{binaryName}", + ["value"] = snapshot.Nodes.Length.ToString(CultureInfo.InvariantCulture) + }); + properties.Add(new JsonObject + { + ["name"] = $"stellaops:callgraph:edgeCount:{binaryName}", + ["value"] = snapshot.Edges.Length.ToString(CultureInfo.InvariantCulture) + }); + properties.Add(new JsonObject + { + ["name"] = $"stellaops:callgraph:entrypointCount:{binaryName}", + ["value"] = snapshot.EntrypointIds.Length.ToString(CultureInfo.InvariantCulture) + }); + properties.Add(new JsonObject + { + ["name"] = $"stellaops:callgraph:extractedAt:{binaryName}", + ["value"] = snapshot.ExtractedAt.ToString("O", CultureInfo.InvariantCulture) + }); + + // Write updated SBOM + var updatedJson = doc.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(sbomPath, updatedJson, cancellationToken); + + return 0; + } + catch + { + return 1; + } + } + private static string DetectFormat(byte[] header) { // ELF magic: 0x7f 'E' 'L' 'F' diff --git a/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs b/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs new file mode 100644 index 000000000..e800d8c48 --- /dev/null +++ b/src/Cli/StellaOps.Cli/Commands/CommandHandlers.Secrets.cs @@ -0,0 +1,429 @@ +// ----------------------------------------------------------------------------- +// CommandHandlers.Secrets.cs +// Sprint: SPRINT_20260104_003_SCANNER (Secret Detection Rule Bundles) +// Tasks: RB-006, RB-007 - Command handlers for secrets bundle operations. +// Description: Implements bundle create, verify, and info commands. +// ----------------------------------------------------------------------------- + +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; + +namespace StellaOps.Cli.Commands; + +internal static partial class CommandHandlers +{ + private static readonly JsonSerializerOptions SecretsJsonOptions = new() + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + internal static async Task HandleSecretsBundleCreateAsync( + IServiceProvider services, + string sources, + string output, + string id, + string? version, + bool sign, + string? keyId, + string? secret, + string? secretFile, + string format, + bool verbose, + CancellationToken cancellationToken) + { + var logger = services.GetService()?.CreateLogger("Secrets.Bundle.Create"); + + if (!Directory.Exists(sources)) + { + AnsiConsole.MarkupLine($"[red]Error: Source directory not found: {Markup.Escape(sources)}[/]"); + return 1; + } + + // Determine version (CalVer if not specified) + var bundleVersion = version ?? DateTimeOffset.UtcNow.ToString("yyyy.MM", System.Globalization.CultureInfo.InvariantCulture); + + if (format == "text") + { + AnsiConsole.MarkupLine("[blue]Creating secrets detection rule bundle...[/]"); + AnsiConsole.MarkupLine($" Source: [bold]{Markup.Escape(sources)}[/]"); + AnsiConsole.MarkupLine($" Output: [bold]{Markup.Escape(output)}[/]"); + AnsiConsole.MarkupLine($" ID: [bold]{Markup.Escape(id)}[/]"); + AnsiConsole.MarkupLine($" Version: [bold]{Markup.Escape(bundleVersion)}[/]"); + } + + try + { + // Create output directory + Directory.CreateDirectory(output); + + // Build the bundle + var builderLogger = services.GetService()?.CreateLogger(); + var validatorLogger = services.GetService()?.CreateLogger(); + + var validator = new RuleValidator(validatorLogger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var builder = new BundleBuilder( + validator, + builderLogger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + var buildOptions = new BundleBuildOptions + { + SourceDirectory = sources, + OutputDirectory = output, + BundleId = id, + Version = bundleVersion, + TimeProvider = TimeProvider.System + }; + + var artifact = await builder.BuildAsync(buildOptions, cancellationToken); + + if (format == "text") + { + AnsiConsole.MarkupLine($"[green]Bundle created successfully![/]"); + AnsiConsole.MarkupLine($" Manifest: [bold]{Markup.Escape(artifact.ManifestPath)}[/]"); + AnsiConsole.MarkupLine($" Rules: [bold]{Markup.Escape(artifact.RulesPath)}[/]"); + AnsiConsole.MarkupLine($" Total rules: [bold]{artifact.TotalRules}[/]"); + AnsiConsole.MarkupLine($" Enabled rules: [bold]{artifact.EnabledRules}[/]"); + AnsiConsole.MarkupLine($" SHA-256: [bold]{artifact.RulesSha256}[/]"); + } + + // Sign if requested + if (sign) + { + if (string.IsNullOrWhiteSpace(keyId)) + { + AnsiConsole.MarkupLine("[red]Error: --key-id is required for signing[/]"); + return 1; + } + + if (string.IsNullOrWhiteSpace(secret) && string.IsNullOrWhiteSpace(secretFile)) + { + AnsiConsole.MarkupLine("[red]Error: --secret or --secret-file is required for signing[/]"); + return 1; + } + + if (format == "text") + { + AnsiConsole.MarkupLine("[blue]Signing bundle...[/]"); + } + + var signerLogger = services.GetService()?.CreateLogger(); + var signer = new BundleSigner(signerLogger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + var signOptions = new BundleSigningOptions + { + KeyId = keyId, + SharedSecret = secret, + SharedSecretFile = secretFile, + TimeProvider = TimeProvider.System + }; + + var signResult = await signer.SignAsync(artifact, signOptions, cancellationToken); + + if (format == "text") + { + AnsiConsole.MarkupLine($"[green]Bundle signed successfully![/]"); + AnsiConsole.MarkupLine($" Envelope: [bold]{Markup.Escape(signResult.EnvelopePath)}[/]"); + AnsiConsole.MarkupLine($" Key ID: [bold]{Markup.Escape(keyId)}[/]"); + } + } + + if (format == "json") + { + var result = new + { + success = true, + bundle = new + { + id, + version = bundleVersion, + manifestPath = artifact.ManifestPath, + rulesPath = artifact.RulesPath, + totalRules = artifact.TotalRules, + enabledRules = artifact.EnabledRules, + rulesSha256 = artifact.RulesSha256, + signed = sign + } + }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + + return 0; + } + catch (Exception ex) + { + if (format == "text") + { + AnsiConsole.MarkupLine($"[red]Error: {Markup.Escape(ex.Message)}[/]"); + if (verbose) + { + AnsiConsole.WriteException(ex); + } + } + else + { + var result = new { success = false, error = ex.Message }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + return 1; + } + } + + internal static async Task HandleSecretsBundleVerifyAsync( + IServiceProvider services, + string bundle, + string? secret, + string? secretFile, + string[] trustedKeys, + bool requireSignature, + string format, + bool verbose, + CancellationToken cancellationToken) + { + if (!Directory.Exists(bundle)) + { + if (format == "text") + { + AnsiConsole.MarkupLine($"[red]Error: Bundle directory not found: {Markup.Escape(bundle)}[/]"); + } + else + { + var result = new { success = false, error = $"Bundle directory not found: {bundle}" }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + return 1; + } + + if (format == "text") + { + AnsiConsole.MarkupLine("[blue]Verifying secrets detection rule bundle...[/]"); + AnsiConsole.MarkupLine($" Bundle: [bold]{Markup.Escape(bundle)}[/]"); + } + + try + { + var verifierLogger = services.GetService()?.CreateLogger(); + var verifier = new BundleVerifier(verifierLogger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + var verifyOptions = new BundleVerificationOptions + { + SharedSecret = secret, + SharedSecretFile = secretFile, + TrustedKeyIds = trustedKeys.Length > 0 ? trustedKeys : null, + SkipSignatureVerification = !requireSignature && string.IsNullOrWhiteSpace(secret) && string.IsNullOrWhiteSpace(secretFile), + VerifyIntegrity = true + }; + + var result = await verifier.VerifyAsync(bundle, verifyOptions, cancellationToken); + + if (format == "text") + { + if (result.IsValid) + { + AnsiConsole.MarkupLine("[green]Bundle verification passed![/]"); + AnsiConsole.MarkupLine($" Bundle ID: [bold]{Markup.Escape(result.BundleId ?? "unknown")}[/]"); + AnsiConsole.MarkupLine($" Version: [bold]{Markup.Escape(result.BundleVersion ?? "unknown")}[/]"); + AnsiConsole.MarkupLine($" Rules: [bold]{result.RuleCount ?? 0}[/]"); + + if (result.SignerKeyId is not null) + { + AnsiConsole.MarkupLine($" Signed by: [bold]{Markup.Escape(result.SignerKeyId)}[/]"); + } + if (result.SignedAt.HasValue) + { + AnsiConsole.MarkupLine($" Signed at: [bold]{result.SignedAt.Value:O}[/]"); + } + + foreach (var warning in result.ValidationWarnings) + { + AnsiConsole.MarkupLine($"[yellow]Warning: {Markup.Escape(warning)}[/]"); + } + } + else + { + AnsiConsole.MarkupLine("[red]Bundle verification failed![/]"); + foreach (var error in result.ValidationErrors) + { + AnsiConsole.MarkupLine($"[red] - {Markup.Escape(error)}[/]"); + } + } + } + else + { + var jsonResult = new + { + success = result.IsValid, + bundleId = result.BundleId, + version = result.BundleVersion, + ruleCount = result.RuleCount, + signerKeyId = result.SignerKeyId, + signedAt = result.SignedAt?.ToString("O"), + errors = result.ValidationErrors, + warnings = result.ValidationWarnings + }; + Console.WriteLine(JsonSerializer.Serialize(jsonResult, SecretsJsonOptions)); + } + + return result.IsValid ? 0 : 1; + } + catch (Exception ex) + { + if (format == "text") + { + AnsiConsole.MarkupLine($"[red]Error: {Markup.Escape(ex.Message)}[/]"); + if (verbose) + { + AnsiConsole.WriteException(ex); + } + } + else + { + var result = new { success = false, error = ex.Message }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + return 1; + } + } + + internal static async Task HandleSecretsBundleInfoAsync( + IServiceProvider services, + string bundle, + string format, + bool verbose, + CancellationToken cancellationToken) + { + var manifestPath = Path.Combine(bundle, "secrets.ruleset.manifest.json"); + + if (!File.Exists(manifestPath)) + { + if (format == "text") + { + AnsiConsole.MarkupLine($"[red]Error: Bundle manifest not found: {Markup.Escape(manifestPath)}[/]"); + } + else + { + var result = new { success = false, error = $"Bundle manifest not found: {manifestPath}" }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + return 1; + } + + try + { + var manifestJson = await File.ReadAllTextAsync(manifestPath, cancellationToken); + var manifest = JsonSerializer.Deserialize(manifestJson, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true + }); + + if (manifest is null) + { + if (format == "text") + { + AnsiConsole.MarkupLine("[red]Error: Failed to parse bundle manifest[/]"); + } + else + { + var result = new { success = false, error = "Failed to parse bundle manifest" }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + return 1; + } + + if (format == "text") + { + AnsiConsole.MarkupLine("[blue]Bundle Information[/]"); + AnsiConsole.MarkupLine($" ID: [bold]{Markup.Escape(manifest.Id)}[/]"); + AnsiConsole.MarkupLine($" Version: [bold]{Markup.Escape(manifest.Version)}[/]"); + AnsiConsole.MarkupLine($" Schema: [bold]{Markup.Escape(manifest.SchemaVersion)}[/]"); + AnsiConsole.MarkupLine($" Created: [bold]{manifest.CreatedAt:O}[/]"); + + if (!string.IsNullOrWhiteSpace(manifest.Description)) + { + AnsiConsole.MarkupLine($" Description: [bold]{Markup.Escape(manifest.Description)}[/]"); + } + + AnsiConsole.MarkupLine(""); + AnsiConsole.MarkupLine("[blue]Integrity[/]"); + AnsiConsole.MarkupLine($" Rules file: [bold]{Markup.Escape(manifest.Integrity.RulesFile)}[/]"); + AnsiConsole.MarkupLine($" SHA-256: [bold]{Markup.Escape(manifest.Integrity.RulesSha256)}[/]"); + AnsiConsole.MarkupLine($" Total rules: [bold]{manifest.Integrity.TotalRules}[/]"); + AnsiConsole.MarkupLine($" Enabled rules: [bold]{manifest.Integrity.EnabledRules}[/]"); + + if (manifest.Signatures is not null) + { + AnsiConsole.MarkupLine(""); + AnsiConsole.MarkupLine("[blue]Signature[/]"); + AnsiConsole.MarkupLine($" Envelope: [bold]{Markup.Escape(manifest.Signatures.DsseEnvelope)}[/]"); + AnsiConsole.MarkupLine($" Key ID: [bold]{Markup.Escape(manifest.Signatures.KeyId ?? "unknown")}[/]"); + if (manifest.Signatures.SignedAt.HasValue) + { + AnsiConsole.MarkupLine($" Signed at: [bold]{manifest.Signatures.SignedAt.Value:O}[/]"); + } + if (!string.IsNullOrWhiteSpace(manifest.Signatures.RekorLogId)) + { + AnsiConsole.MarkupLine($" Rekor log ID: [bold]{Markup.Escape(manifest.Signatures.RekorLogId)}[/]"); + } + } + else + { + AnsiConsole.MarkupLine(""); + AnsiConsole.MarkupLine("[yellow]Bundle is not signed[/]"); + } + + if (verbose && manifest.Rules.Length > 0) + { + AnsiConsole.MarkupLine(""); + AnsiConsole.MarkupLine("[blue]Rules Summary[/]"); + var table = new Table(); + table.AddColumn("ID"); + table.AddColumn("Version"); + table.AddColumn("Severity"); + table.AddColumn("Enabled"); + + foreach (var rule in manifest.Rules.Take(20)) + { + table.AddRow( + Markup.Escape(rule.Id), + Markup.Escape(rule.Version), + Markup.Escape(rule.Severity), + rule.Enabled ? "[green]Yes[/]" : "[red]No[/]"); + } + + if (manifest.Rules.Length > 20) + { + table.AddRow($"... and {manifest.Rules.Length - 20} more", "", "", ""); + } + + AnsiConsole.Write(table); + } + } + else + { + Console.WriteLine(JsonSerializer.Serialize(manifest, SecretsJsonOptions)); + } + + return 0; + } + catch (Exception ex) + { + if (format == "text") + { + AnsiConsole.MarkupLine($"[red]Error: {Markup.Escape(ex.Message)}[/]"); + if (verbose) + { + AnsiConsole.WriteException(ex); + } + } + else + { + var result = new { success = false, error = ex.Message }; + Console.WriteLine(JsonSerializer.Serialize(result, SecretsJsonOptions)); + } + return 1; + } + } +} diff --git a/src/Cli/StellaOps.Cli/Commands/SecretsCommandGroup.cs b/src/Cli/StellaOps.Cli/Commands/SecretsCommandGroup.cs new file mode 100644 index 000000000..12cd922fb --- /dev/null +++ b/src/Cli/StellaOps.Cli/Commands/SecretsCommandGroup.cs @@ -0,0 +1,247 @@ +// ----------------------------------------------------------------------------- +// SecretsCommandGroup.cs +// Sprint: SPRINT_20260104_003_SCANNER (Secret Detection Rule Bundles) +// Tasks: RB-006, RB-007 - CLI commands for secrets rule bundle management. +// Description: CLI commands for building, signing, and verifying secrets bundles. +// ----------------------------------------------------------------------------- + +using System.CommandLine; +using StellaOps.Cli.Extensions; + +namespace StellaOps.Cli.Commands; + +internal static class SecretsCommandGroup +{ + internal static Command BuildSecretsCommand( + IServiceProvider services, + Option verboseOption, + CancellationToken cancellationToken) + { + var secrets = new Command("secrets", "Secrets detection rule bundle management."); + + secrets.Add(BuildBundleCommand(services, verboseOption, cancellationToken)); + + return secrets; + } + + private static Command BuildBundleCommand( + IServiceProvider services, + Option verboseOption, + CancellationToken cancellationToken) + { + var bundle = new Command("bundle", "Secrets rule bundle operations."); + + bundle.Add(BuildCreateCommand(services, verboseOption, cancellationToken)); + bundle.Add(BuildVerifyCommand(services, verboseOption, cancellationToken)); + bundle.Add(BuildInfoCommand(services, verboseOption, cancellationToken)); + + return bundle; + } + + private static Command BuildCreateCommand( + IServiceProvider services, + Option verboseOption, + CancellationToken cancellationToken) + { + var sourcesArg = new Argument("sources") + { + Description = "Path to directory containing rule JSON files" + }; + + var outputOption = new Option("--output", "-o") + { + Description = "Output directory for the bundle (default: ./bundle)" + }; + outputOption.SetDefaultValue("./bundle"); + + var idOption = new Option("--id") + { + Description = "Bundle identifier (default: stellaops-secrets)" + }; + idOption.SetDefaultValue("stellaops-secrets"); + + var versionOption = new Option("--version", "-v") + { + Description = "Bundle version (CalVer, e.g., 2026.01)" + }; + + var signOption = new Option("--sign") + { + Description = "Sign the bundle after creation" + }; + + var keyIdOption = new Option("--key-id") + { + Description = "Key identifier for signing" + }; + + var secretOption = new Option("--secret") + { + Description = "Shared secret for HMAC signing (base64 or hex)" + }; + + var secretFileOption = new Option("--secret-file") + { + Description = "Path to file containing the shared secret" + }; + + var formatOption = new Option("--format") + { + Description = "Output format: text, json" + }.SetDefaultValue("text").FromAmong("text", "json"); + + var command = new Command("create", "Create a secrets detection rule bundle from JSON rule definitions.") + { + sourcesArg, + outputOption, + idOption, + versionOption, + signOption, + keyIdOption, + secretOption, + secretFileOption, + formatOption, + verboseOption + }; + + command.SetAction(parseResult => + { + var sources = parseResult.GetValue(sourcesArg) ?? string.Empty; + var output = parseResult.GetValue(outputOption) ?? "./bundle"; + var id = parseResult.GetValue(idOption) ?? "stellaops-secrets"; + var version = parseResult.GetValue(versionOption); + var sign = parseResult.GetValue(signOption); + var keyId = parseResult.GetValue(keyIdOption); + var secret = parseResult.GetValue(secretOption); + var secretFile = parseResult.GetValue(secretFileOption); + var format = parseResult.GetValue(formatOption) ?? "text"; + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleSecretsBundleCreateAsync( + services, + sources, + output, + id, + version, + sign, + keyId, + secret, + secretFile, + format, + verbose, + cancellationToken); + }); + + return command; + } + + private static Command BuildVerifyCommand( + IServiceProvider services, + Option verboseOption, + CancellationToken cancellationToken) + { + var bundleArg = new Argument("bundle") + { + Description = "Path to the bundle directory" + }; + + var secretOption = new Option("--secret") + { + Description = "Shared secret for HMAC verification (base64 or hex)" + }; + + var secretFileOption = new Option("--secret-file") + { + Description = "Path to file containing the shared secret" + }; + + var trustedKeysOption = new Option("--trusted-keys") + { + Description = "List of trusted key IDs for signature verification" + }; + + var requireSignatureOption = new Option("--require-signature") + { + Description = "Require a valid signature (fail if unsigned)" + }; + + var formatOption = new Option("--format") + { + Description = "Output format: text, json" + }.SetDefaultValue("text").FromAmong("text", "json"); + + var command = new Command("verify", "Verify a secrets detection rule bundle's integrity and signature.") + { + bundleArg, + secretOption, + secretFileOption, + trustedKeysOption, + requireSignatureOption, + formatOption, + verboseOption + }; + + command.SetAction(parseResult => + { + var bundle = parseResult.GetValue(bundleArg) ?? string.Empty; + var secret = parseResult.GetValue(secretOption); + var secretFile = parseResult.GetValue(secretFileOption); + var trustedKeys = parseResult.GetValue(trustedKeysOption) ?? Array.Empty(); + var requireSignature = parseResult.GetValue(requireSignatureOption); + var format = parseResult.GetValue(formatOption) ?? "text"; + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleSecretsBundleVerifyAsync( + services, + bundle, + secret, + secretFile, + trustedKeys, + requireSignature, + format, + verbose, + cancellationToken); + }); + + return command; + } + + private static Command BuildInfoCommand( + IServiceProvider services, + Option verboseOption, + CancellationToken cancellationToken) + { + var bundleArg = new Argument("bundle") + { + Description = "Path to the bundle directory" + }; + + var formatOption = new Option("--format") + { + Description = "Output format: text, json" + }.SetDefaultValue("text").FromAmong("text", "json"); + + var command = new Command("info", "Display information about a secrets detection rule bundle.") + { + bundleArg, + formatOption, + verboseOption + }; + + command.SetAction(parseResult => + { + var bundle = parseResult.GetValue(bundleArg) ?? string.Empty; + var format = parseResult.GetValue(formatOption) ?? "text"; + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleSecretsBundleInfoAsync( + services, + bundle, + format, + verbose, + cancellationToken); + }); + + return command; + } +} diff --git a/src/Cli/StellaOps.Cli/StellaOps.Cli.csproj b/src/Cli/StellaOps.Cli/StellaOps.Cli.csproj index d20127860..038f1a6fb 100644 --- a/src/Cli/StellaOps.Cli/StellaOps.Cli.csproj +++ b/src/Cli/StellaOps.Cli/StellaOps.Cli.csproj @@ -90,6 +90,10 @@ + + + + diff --git a/src/Policy/StellaOps.Policy.Engine/AGENTS.md b/src/Policy/StellaOps.Policy.Engine/AGENTS.md index e1040da24..0eef909d0 100644 --- a/src/Policy/StellaOps.Policy.Engine/AGENTS.md +++ b/src/Policy/StellaOps.Policy.Engine/AGENTS.md @@ -9,6 +9,10 @@ Stand up the Policy Engine runtime host that evaluates organization policies aga - Change stream listeners and scheduler integration for incremental re-evaluation. - Authority integration enforcing new `policy:*` and `effective:write` scopes. - Observability: metrics, traces, structured logs, trace sampling. +- **StabilityDampingGate** (Sprint NG-001): Hysteresis-based damping to prevent flip-flopping verdicts: + - Suppresses rapid status oscillations requiring min duration or significant confidence delta + - Upgrades (more severe) bypass damping; downgrades are dampable + - Integrates with VexLens NoiseGate for unified noise-gating ## Expectations - Keep endpoints deterministic, cancellation-aware, and tenant-scoped. diff --git a/src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingGate.cs b/src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingGate.cs new file mode 100644 index 000000000..8e383d3b2 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingGate.cs @@ -0,0 +1,384 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Concurrent; +using System.Globalization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Policy.Engine.Gates; + +/// +/// Gate that applies hysteresis-based stability damping to prevent flip-flopping verdicts. +/// +/// +/// This gate tracks verdict state transitions and suppresses rapid oscillations by requiring: +/// - A minimum duration before a state change is surfaced, OR +/// - A significant confidence delta that justifies immediate surfacing +/// +/// This reduces alert fatigue from noisy or unstable feed data while still ensuring +/// significant changes (especially upgrades to more severe states) surface promptly. +/// +public interface IStabilityDampingGate +{ + /// + /// Evaluates whether a verdict transition should be surfaced or damped. + /// + /// The damping evaluation request. + /// Cancellation token. + /// The damping decision. + Task EvaluateAsync( + StabilityDampingRequest request, + CancellationToken cancellationToken = default); + + /// + /// Records a verdict state for future damping calculations. + /// + /// The unique key for this verdict (e.g., "artifact:cve"). + /// The verdict state to record. + /// Cancellation token. + Task RecordStateAsync( + string key, + VerdictState state, + CancellationToken cancellationToken = default); + + /// + /// Prunes old verdict history records. + /// + /// Cancellation token. + /// Number of records pruned. + Task PruneHistoryAsync(CancellationToken cancellationToken = default); +} + +/// +/// Request for stability damping evaluation. +/// +public sealed record StabilityDampingRequest +{ + /// + /// Gets the unique key for this verdict (e.g., "artifact:cve" or "sha256:vuln_id"). + /// + public required string Key { get; init; } + + /// + /// Gets the current (proposed) verdict state. + /// + public required VerdictState ProposedState { get; init; } + + /// + /// Gets the tenant ID for multi-tenant deployments. + /// + public string? TenantId { get; init; } +} + +/// +/// Represents a verdict state at a point in time. +/// +public sealed record VerdictState +{ + /// + /// Gets the VEX status (affected, not_affected, fixed, under_investigation). + /// + public required string Status { get; init; } + + /// + /// Gets the confidence score (0.0 to 1.0). + /// + public required double Confidence { get; init; } + + /// + /// Gets the timestamp of this state. + /// + public required DateTimeOffset Timestamp { get; init; } + + /// + /// Gets the rationale class (e.g., "authoritative", "binary", "static"). + /// + public string? RationaleClass { get; init; } + + /// + /// Gets the source that produced this state. + /// + public string? SourceId { get; init; } +} + +/// +/// Decision from stability damping evaluation. +/// +public sealed record StabilityDampingDecision +{ + /// + /// Gets whether the transition should be surfaced. + /// + public required bool ShouldSurface { get; init; } + + /// + /// Gets the reason for the decision. + /// + public required string Reason { get; init; } + + /// + /// Gets the previous state, if any. + /// + public VerdictState? PreviousState { get; init; } + + /// + /// Gets how long the previous state has persisted. + /// + public TimeSpan? StateDuration { get; init; } + + /// + /// Gets the confidence delta from previous to current. + /// + public double? ConfidenceDelta { get; init; } + + /// + /// Gets whether this is a status upgrade (more severe). + /// + public bool? IsUpgrade { get; init; } + + /// + /// Gets the timestamp of the decision. + /// + public required DateTimeOffset DecidedAt { get; init; } +} + +/// +/// Default implementation of . +/// +public sealed class StabilityDampingGate : IStabilityDampingGate +{ + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly ConcurrentDictionary _stateHistory = new(); + + // Status severity ordering: higher = more severe + private static readonly Dictionary StatusSeverity = new(StringComparer.OrdinalIgnoreCase) + { + ["affected"] = 100, + ["under_investigation"] = 50, + ["fixed"] = 25, + ["not_affected"] = 0 + }; + + /// + /// Initializes a new instance of the class. + /// + public StabilityDampingGate( + IOptionsMonitor options, + TimeProvider timeProvider, + ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public Task EvaluateAsync( + StabilityDampingRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var opts = _options.CurrentValue; + var now = _timeProvider.GetUtcNow(); + var key = BuildKey(request.TenantId, request.Key); + + // If disabled, always surface + if (!opts.Enabled) + { + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = "Stability damping disabled", + DecidedAt = now + }); + } + + // Check if status is subject to damping + if (!opts.DampedStatuses.Contains(request.ProposedState.Status)) + { + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Status '{request.ProposedState.Status}' is not subject to damping"), + DecidedAt = now + }); + } + + // Get previous state + if (!_stateHistory.TryGetValue(key, out var previousState)) + { + // No history - this is a new verdict, surface it + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = "No previous state (new verdict)", + DecidedAt = now + }); + } + + // Check if status actually changed + if (string.Equals(previousState.Status, request.ProposedState.Status, StringComparison.OrdinalIgnoreCase)) + { + // Same status - check confidence delta + var confidenceDelta = Math.Abs(request.ProposedState.Confidence - previousState.Confidence); + + if (confidenceDelta >= opts.MinConfidenceDeltaPercent) + { + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Confidence changed by {confidenceDelta:P1} (threshold: {opts.MinConfidenceDeltaPercent:P1})"), + PreviousState = previousState, + ConfidenceDelta = confidenceDelta, + DecidedAt = now + }); + } + + // No significant change + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = false, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Same status, confidence delta {confidenceDelta:P1} below threshold"), + PreviousState = previousState, + ConfidenceDelta = confidenceDelta, + DecidedAt = now + }); + } + + // Status changed - check if it's an upgrade or downgrade + var isUpgrade = IsStatusUpgrade(previousState.Status, request.ProposedState.Status); + var stateDuration = now - previousState.Timestamp; + + // Upgrades (more severe) bypass damping if configured + if (isUpgrade && opts.OnlyDampDowngrades) + { + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Status upgrade ({previousState.Status} -> {request.ProposedState.Status}) surfaces immediately"), + PreviousState = previousState, + StateDuration = stateDuration, + IsUpgrade = true, + DecidedAt = now + }); + } + + // Check confidence delta for immediate surfacing + var delta = Math.Abs(request.ProposedState.Confidence - previousState.Confidence); + if (delta >= opts.MinConfidenceDeltaPercent) + { + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Confidence delta {delta:P1} exceeds threshold {opts.MinConfidenceDeltaPercent:P1}"), + PreviousState = previousState, + StateDuration = stateDuration, + ConfidenceDelta = delta, + IsUpgrade = isUpgrade, + DecidedAt = now + }); + } + + // Check duration requirement + if (stateDuration >= opts.MinDurationBeforeChange) + { + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = true, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Previous state persisted for {stateDuration.TotalHours:F1}h (threshold: {opts.MinDurationBeforeChange.TotalHours:F1}h)"), + PreviousState = previousState, + StateDuration = stateDuration, + IsUpgrade = isUpgrade, + DecidedAt = now + }); + } + + // Damped - don't surface yet + var remainingTime = opts.MinDurationBeforeChange - stateDuration; + + if (opts.LogDampedTransitions) + { + _logger.LogDebug( + "Damped transition for {Key}: {OldStatus}->{NewStatus}, remaining: {Remaining}", + request.Key, + previousState.Status, + request.ProposedState.Status, + remainingTime); + } + + return Task.FromResult(new StabilityDampingDecision + { + ShouldSurface = false, + Reason = string.Create(CultureInfo.InvariantCulture, + $"Damped: state duration {stateDuration.TotalHours:F1}h < {opts.MinDurationBeforeChange.TotalHours:F1}h, " + + $"delta {delta:P1} < {opts.MinConfidenceDeltaPercent:P1}"), + PreviousState = previousState, + StateDuration = stateDuration, + ConfidenceDelta = delta, + IsUpgrade = isUpgrade, + DecidedAt = now + }); + } + + /// + public Task RecordStateAsync( + string key, + VerdictState state, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(state); + + _stateHistory[key] = state; + return Task.CompletedTask; + } + + /// + public Task PruneHistoryAsync(CancellationToken cancellationToken = default) + { + var opts = _options.CurrentValue; + var cutoff = _timeProvider.GetUtcNow() - opts.HistoryRetention; + var pruned = 0; + + foreach (var kvp in _stateHistory) + { + if (kvp.Value.Timestamp < cutoff) + { + if (_stateHistory.TryRemove(kvp.Key, out _)) + { + pruned++; + } + } + } + + if (pruned > 0) + { + _logger.LogInformation("Pruned {Count} stale verdict state records", pruned); + } + + return Task.FromResult(pruned); + } + + private static string BuildKey(string? tenantId, string verdictKey) + { + return string.IsNullOrEmpty(tenantId) + ? verdictKey + : $"{tenantId}:{verdictKey}"; + } + + private static bool IsStatusUpgrade(string oldStatus, string newStatus) + { + var oldSeverity = StatusSeverity.GetValueOrDefault(oldStatus, 50); + var newSeverity = StatusSeverity.GetValueOrDefault(newStatus, 50); + return newSeverity > oldSeverity; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingOptions.cs b/src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingOptions.cs new file mode 100644 index 000000000..bbfa02952 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Gates/StabilityDampingOptions.cs @@ -0,0 +1,81 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Policy.Engine.Gates; + +/// +/// Configuration options for the stability damping gate. +/// +/// +/// Stability damping prevents flip-flopping verdicts by requiring that: +/// - A verdict must persist for a minimum duration before a change is surfaced, OR +/// - The confidence delta must exceed a minimum threshold +/// +/// This reduces notification noise from unstable feed data while still allowing +/// significant changes to surface quickly. +/// +public sealed class StabilityDampingOptions +{ + /// + /// Gets or sets whether stability damping is enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Gets or sets the minimum duration a verdict must persist before + /// a change is surfaced, unless the confidence delta exceeds the threshold. + /// + /// + /// Default: 4 hours. Set to TimeSpan.Zero to disable duration-based damping. + /// + [Range(typeof(TimeSpan), "00:00:00", "7.00:00:00", + ErrorMessage = "MinDurationBeforeChange must be between 0 and 7 days.")] + public TimeSpan MinDurationBeforeChange { get; set; } = TimeSpan.FromHours(4); + + /// + /// Gets or sets the minimum confidence change percentage required to + /// bypass the duration requirement and surface a change immediately. + /// + /// + /// Default: 15%. A change from 0.70 to 0.85 (15%) would bypass duration damping. + /// Set to 1.0 (100%) to effectively disable delta-based bypass. + /// + [Range(0.0, 1.0, ErrorMessage = "MinConfidenceDeltaPercent must be between 0 and 1.")] + public double MinConfidenceDeltaPercent { get; set; } = 0.15; + + /// + /// Gets or sets the VEX statuses to which damping applies. + /// + /// + /// By default, damping applies to affected and not_affected transitions. + /// Transitions to/from under_investigation are typically not damped. + /// + public HashSet DampedStatuses { get; set; } = + [ + "affected", + "not_affected", + "fixed" + ]; + + /// + /// Gets or sets whether to apply damping only to downgrade transitions + /// (e.g., affected -> not_affected). + /// + /// + /// When true, upgrades (not_affected -> affected) are surfaced immediately. + /// This is conservative: users are alerted to new risks without delay. + /// + public bool OnlyDampDowngrades { get; set; } = true; + + /// + /// Gets or sets the retention period for verdict history. + /// Older records are pruned to prevent unbounded growth. + /// + public TimeSpan HistoryRetention { get; set; } = TimeSpan.FromDays(30); + + /// + /// Gets or sets whether to log damped transitions for debugging. + /// + public bool LogDampedTransitions { get; set; } = false; +} diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/StabilityDampingGateTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/StabilityDampingGateTests.cs new file mode 100644 index 000000000..c396acf82 --- /dev/null +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/StabilityDampingGateTests.cs @@ -0,0 +1,347 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Policy.Engine.Gates; +using Xunit; + +namespace StellaOps.Policy.Engine.Tests.Gates; + +/// +/// Unit tests for . +/// +[Trait("Category", "Unit")] +public class StabilityDampingGateTests +{ + private readonly FakeTimeProvider _timeProvider; + private readonly StabilityDampingOptions _defaultOptions; + + public StabilityDampingGateTests() + { + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + _defaultOptions = new StabilityDampingOptions + { + Enabled = true, + MinDurationBeforeChange = TimeSpan.FromHours(4), + MinConfidenceDeltaPercent = 0.15, + OnlyDampDowngrades = true, + DampedStatuses = ["affected", "not_affected", "fixed", "under_investigation"] + }; + } + + private StabilityDampingGate CreateGate(StabilityDampingOptions? options = null) + { + var opts = options ?? _defaultOptions; + var optionsMonitor = new TestOptionsMonitor(opts); + return new StabilityDampingGate(optionsMonitor, _timeProvider, NullLogger.Instance); + } + + [Fact] + public async Task EvaluateAsync_NewVerdict_ShouldSurface() + { + // Arrange + var gate = CreateGate(); + var request = new StabilityDampingRequest + { + Key = "artifact:CVE-2024-1234", + ProposedState = new VerdictState + { + Status = "affected", + Confidence = 0.85, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeTrue(); + decision.Reason.Should().Contain("new verdict"); + } + + [Fact] + public async Task EvaluateAsync_WhenDisabled_ShouldAlwaysSurface() + { + // Arrange + var options = new StabilityDampingOptions { Enabled = false }; + var gate = CreateGate(options); + var request = new StabilityDampingRequest + { + Key = "artifact:CVE-2024-1234", + ProposedState = new VerdictState + { + Status = "affected", + Confidence = 0.85, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeTrue(); + decision.Reason.Should().Contain("disabled"); + } + + [Fact] + public async Task EvaluateAsync_StatusUpgrade_ShouldSurfaceImmediately() + { + // Arrange + var gate = CreateGate(); + var key = "artifact:CVE-2024-1234"; + + // Record initial state as not_affected + await gate.RecordStateAsync(key, new VerdictState + { + Status = "not_affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Request upgrade to affected (more severe) + var request = new StabilityDampingRequest + { + Key = key, + ProposedState = new VerdictState + { + Status = "affected", + Confidence = 0.85, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeTrue(); + decision.IsUpgrade.Should().BeTrue(); + decision.Reason.Should().Contain("upgrade"); + } + + [Fact] + public async Task EvaluateAsync_StatusDowngrade_WithoutMinDuration_ShouldDamp() + { + // Arrange + var gate = CreateGate(); + var key = "artifact:CVE-2024-1234"; + + // Record initial state as affected + await gate.RecordStateAsync(key, new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Advance time but not enough to meet threshold + _timeProvider.Advance(TimeSpan.FromHours(2)); + + // Request downgrade to not_affected (less severe) + var request = new StabilityDampingRequest + { + Key = key, + ProposedState = new VerdictState + { + Status = "not_affected", + Confidence = 0.75, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeFalse(); + decision.Reason.Should().Contain("Damped"); + } + + [Fact] + public async Task EvaluateAsync_StatusDowngrade_AfterMinDuration_ShouldSurface() + { + // Arrange + var gate = CreateGate(); + var key = "artifact:CVE-2024-1234"; + + // Record initial state as affected + await gate.RecordStateAsync(key, new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Advance time past threshold + _timeProvider.Advance(TimeSpan.FromHours(5)); + + // Request downgrade to not_affected + var request = new StabilityDampingRequest + { + Key = key, + ProposedState = new VerdictState + { + Status = "not_affected", + Confidence = 0.75, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeTrue(); + decision.StateDuration.Should().BeGreaterThan(TimeSpan.FromHours(4)); + } + + [Fact] + public async Task EvaluateAsync_LargeConfidenceDelta_ShouldSurfaceImmediately() + { + // Arrange + var gate = CreateGate(); + var key = "artifact:CVE-2024-1234"; + + // Record initial state + await gate.RecordStateAsync(key, new VerdictState + { + Status = "affected", + Confidence = 0.50, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Request with large confidence change (>15%) + var request = new StabilityDampingRequest + { + Key = key, + ProposedState = new VerdictState + { + Status = "affected", + Confidence = 0.90, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeTrue(); + decision.ConfidenceDelta.Should().BeGreaterThan(0.15); + } + + [Fact] + public async Task EvaluateAsync_SmallConfidenceDelta_SameStatus_ShouldDamp() + { + // Arrange + var gate = CreateGate(); + var key = "artifact:CVE-2024-1234"; + + // Record initial state + await gate.RecordStateAsync(key, new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Request with small confidence change (<15%) + var request = new StabilityDampingRequest + { + Key = key, + ProposedState = new VerdictState + { + Status = "affected", + Confidence = 0.85, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeFalse(); + decision.ConfidenceDelta.Should().BeLessThan(0.15); + } + + [Fact] + public async Task PruneHistoryAsync_ShouldRemoveStaleRecords() + { + // Arrange + var gate = CreateGate(); + + // Record old state + await gate.RecordStateAsync("old-key", new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Advance time past retention period + _timeProvider.Advance(TimeSpan.FromDays(8)); // Default retention is 7 days + + // Record new state (to ensure we have something current) + await gate.RecordStateAsync("new-key", new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Act + var pruned = await gate.PruneHistoryAsync(); + + // Assert + pruned.Should().BeGreaterThanOrEqualTo(1); + } + + [Fact] + public async Task EvaluateAsync_WithTenantId_ShouldIsolateTenants() + { + // Arrange + var gate = CreateGate(); + + // Record state for tenant-a + await gate.RecordStateAsync("tenant-a:artifact:CVE-2024-1234", new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + }); + + // Request for tenant-b (different tenant, no history) + var request = new StabilityDampingRequest + { + Key = "artifact:CVE-2024-1234", + TenantId = "tenant-b", + ProposedState = new VerdictState + { + Status = "affected", + Confidence = 0.80, + Timestamp = _timeProvider.GetUtcNow() + } + }; + + // Act + var decision = await gate.EvaluateAsync(request); + + // Assert + decision.ShouldSurface.Should().BeTrue(); + decision.Reason.Should().Contain("new verdict"); + } + + private sealed class TestOptionsMonitor : IOptionsMonitor + where T : class + { + public TestOptionsMonitor(T value) => CurrentValue = value; + public T CurrentValue { get; } + public T Get(string? name) => CurrentValue; + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs b/src/Scanner/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs index 3bb73076f..525b89dc7 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs @@ -25,6 +25,12 @@ public sealed class ScannerWorkerMetrics private readonly Counter _surfacePayloadPersisted; private readonly Histogram _surfaceManifestPublishDurationMs; + // Secrets analysis metrics (Sprint: SPRINT_20251229_046_BE) + private readonly Counter _secretsAnalysisCompleted; + private readonly Counter _secretsAnalysisFailed; + private readonly Counter _secretFindingsDetected; + private readonly Histogram _secretsAnalysisDurationMs; + public ScannerWorkerMetrics() { _queueLatencyMs = ScannerWorkerInstrumentation.Meter.CreateHistogram( @@ -80,6 +86,21 @@ public sealed class ScannerWorkerMetrics "scanner_worker_surface_manifest_publish_duration_ms", unit: "ms", description: "Duration in milliseconds to persist and publish surface manifests."); + + // Secrets analysis metrics (Sprint: SPRINT_20251229_046_BE) + _secretsAnalysisCompleted = ScannerWorkerInstrumentation.Meter.CreateCounter( + "scanner_worker_secrets_analysis_completed_total", + description: "Number of successfully completed secrets analysis runs."); + _secretsAnalysisFailed = ScannerWorkerInstrumentation.Meter.CreateCounter( + "scanner_worker_secrets_analysis_failed_total", + description: "Number of secrets analysis runs that failed."); + _secretFindingsDetected = ScannerWorkerInstrumentation.Meter.CreateCounter( + "scanner_worker_secrets_findings_detected_total", + description: "Number of secret findings detected."); + _secretsAnalysisDurationMs = ScannerWorkerInstrumentation.Meter.CreateHistogram( + "scanner_worker_secrets_analysis_duration_ms", + unit: "ms", + description: "Duration in milliseconds for secrets analysis."); } public void RecordQueueLatency(ScanJobContext context, TimeSpan latency) @@ -343,4 +364,39 @@ public sealed class ScannerWorkerMetrics // Native analysis metrics are tracked via counters/histograms // This is a placeholder for when we add dedicated native analysis metrics } + + /// + /// Records successful secrets analysis completion. + /// Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection + /// + public void RecordSecretsAnalysisCompleted( + ScanJobContext context, + int findingCount, + int filesScanned, + TimeSpan duration, + TimeProvider timeProvider) + { + var tags = CreateTags(context, stage: ScanStageNames.ScanSecrets); + _secretsAnalysisCompleted.Add(1, tags); + + if (findingCount > 0) + { + _secretFindingsDetected.Add(findingCount, tags); + } + + if (duration > TimeSpan.Zero) + { + _secretsAnalysisDurationMs.Record(duration.TotalMilliseconds, tags); + } + } + + /// + /// Records secrets analysis failure. + /// Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection + /// + public void RecordSecretsAnalysisFailed(ScanJobContext context, TimeProvider timeProvider) + { + var tags = CreateTags(context, stage: ScanStageNames.ScanSecrets); + _secretsAnalysisFailed.Add(1, tags); + } } diff --git a/src/Scanner/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs b/src/Scanner/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs index 914d6f350..e4c3fa006 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs @@ -38,6 +38,12 @@ public sealed class ScannerWorkerOptions public VerdictPushOptions VerdictPush { get; } = new(); + /// + /// Options for secrets leak detection scanning. + /// Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection + /// + public SecretsOptions Secrets { get; } = new(); + public sealed class QueueOptions { public int MaxAttempts { get; set; } = 5; @@ -311,4 +317,43 @@ public sealed class ScannerWorkerOptions /// public bool AllowAnonymousFallback { get; set; } = true; } + + /// + /// Options for secrets leak detection scanning. + /// Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection + /// + public sealed class SecretsOptions + { + /// + /// Enable secrets leak detection scanning. + /// When disabled, the secrets scan stage will be skipped. + /// + public bool Enabled { get; set; } + + /// + /// Path to the secrets ruleset bundle directory. + /// + public string RulesetPath { get; set; } = string.Empty; + + /// + /// Maximum file size in bytes to scan for secrets. + /// Files larger than this will be skipped. + /// + public long MaxFileSizeBytes { get; set; } = 5 * 1024 * 1024; // 5 MB + + /// + /// Maximum number of files to scan per job. + /// + public int MaxFilesPerJob { get; set; } = 10_000; + + /// + /// Enable entropy-based secret detection. + /// + public bool EnableEntropyDetection { get; set; } = true; + + /// + /// Minimum entropy threshold for high-entropy string detection. + /// + public double EntropyThreshold { get; set; } = 4.5; + } } diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs index 6752c11de..c8fd4b617 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs @@ -23,6 +23,9 @@ public static class ScanStageNames // Sprint: SPRINT_20251226_014_BINIDX - Binary Vulnerability Lookup public const string BinaryLookup = "binary-lookup"; + // Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection + public const string ScanSecrets = "scan-secrets"; + public static readonly IReadOnlyList Ordered = new[] { IngestReplay, @@ -30,6 +33,7 @@ public static class ScanStageNames PullLayers, BuildFilesystem, ExecuteAnalyzers, + ScanSecrets, BinaryLookup, EpssEnrichment, ComposeArtifacts, diff --git a/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs b/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs new file mode 100644 index 000000000..d66ada607 --- /dev/null +++ b/src/Scanner/StellaOps.Scanner.Worker/Processing/Secrets/SecretsAnalyzerStageExecutor.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Analyzers.Secrets; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Worker.Diagnostics; +using StellaOps.Scanner.Worker.Options; + +namespace StellaOps.Scanner.Worker.Processing.Secrets; + +/// +/// Stage executor that scans filesystem for hardcoded secrets and credentials. +/// +internal sealed class SecretsAnalyzerStageExecutor : IScanStageExecutor +{ + private static readonly string[] RootFsMetadataKeys = + { + "filesystem.rootfs", + "rootfs.path", + "scanner.rootfs", + }; + + private readonly ISecretsAnalyzer _secretsAnalyzer; + private readonly ScannerWorkerMetrics _metrics; + private readonly TimeProvider _timeProvider; + private readonly IOptions _options; + private readonly ILogger _logger; + + public SecretsAnalyzerStageExecutor( + ISecretsAnalyzer secretsAnalyzer, + ScannerWorkerMetrics metrics, + TimeProvider timeProvider, + IOptions options, + ILogger logger) + { + _secretsAnalyzer = secretsAnalyzer ?? throw new ArgumentNullException(nameof(secretsAnalyzer)); + _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string StageName => ScanStageNames.ScanSecrets; + + public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var secretsOptions = _options.Value.Secrets; + if (!secretsOptions.Enabled) + { + _logger.LogDebug("Secrets scanning is disabled; skipping stage."); + return; + } + + // Get file entries from analyzer stage + if (!context.Analysis.TryGet>(ScanAnalysisKeys.FileEntries, out var files) || files is null) + { + _logger.LogDebug("No file entries available; skipping secrets scan."); + return; + } + + var rootfsPath = ResolveRootfsPath(context.Lease.Metadata); + if (string.IsNullOrWhiteSpace(rootfsPath)) + { + _logger.LogWarning("No rootfs path found in job metadata; skipping secrets scan for job {JobId}.", context.JobId); + return; + } + + var startTime = _timeProvider.GetTimestamp(); + var allFindings = new List(); + + try + { + // Filter to text-like files only + var textFiles = files + .Where(f => ShouldScanFile(f)) + .ToList(); + + _logger.LogInformation( + "Scanning {FileCount} files for secrets in job {JobId}.", + textFiles.Count, + context.JobId); + + foreach (var file in textFiles) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var filePath = Path.Combine(rootfsPath, file.Path.TrimStart('/')); + if (!File.Exists(filePath)) + { + continue; + } + + var content = await File.ReadAllBytesAsync(filePath, cancellationToken).ConfigureAwait(false); + if (content.Length == 0 || content.Length > secretsOptions.MaxFileSizeBytes) + { + continue; + } + + var findings = await _secretsAnalyzer.AnalyzeAsync( + content, + file.Path, + cancellationToken).ConfigureAwait(false); + + if (findings.Count > 0) + { + allFindings.AddRange(findings); + } + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogDebug(ex, "Error scanning file {Path} for secrets: {Message}", file.Path, ex.Message); + } + } + + var elapsed = _timeProvider.GetElapsedTime(startTime); + + // Store findings in analysis store + var report = new SecretsAnalysisReport + { + JobId = context.JobId, + ScanId = context.ScanId, + Findings = allFindings.ToImmutableArray(), + FilesScanned = textFiles.Count, + RulesetVersion = _secretsAnalyzer.RulesetVersion, + AnalyzedAtUtc = _timeProvider.GetUtcNow(), + ElapsedMilliseconds = elapsed.TotalMilliseconds + }; + + context.Analysis.Set(ScanAnalysisKeys.SecretFindings, report); + context.Analysis.Set(ScanAnalysisKeys.SecretRulesetVersion, _secretsAnalyzer.RulesetVersion); + + _metrics.RecordSecretsAnalysisCompleted( + context, + allFindings.Count, + textFiles.Count, + elapsed, + _timeProvider); + + _logger.LogInformation( + "Secrets scan completed for job {JobId}: {FindingCount} findings in {FileCount} files ({ElapsedMs:F0}ms).", + context.JobId, + allFindings.Count, + textFiles.Count, + elapsed.TotalMilliseconds); + } + catch (OperationCanceledException) + { + _logger.LogDebug("Secrets scan cancelled for job {JobId}.", context.JobId); + throw; + } + catch (Exception ex) + { + _metrics.RecordSecretsAnalysisFailed(context, _timeProvider); + _logger.LogError(ex, "Secrets scan failed for job {JobId}: {Message}", context.JobId, ex.Message); + } + } + + private static bool ShouldScanFile(ScanFileEntry file) + { + if (file is null || file.SizeBytes == 0) + { + return false; + } + + // Skip binary files + if (file.Kind is "elf" or "pe" or "mach-o" or "blob") + { + return false; + } + + // Skip very large files + if (file.SizeBytes > 10 * 1024 * 1024) + { + return false; + } + + var ext = Path.GetExtension(file.Path).ToLowerInvariant(); + + // Include common text/config file extensions + return ext is ".json" or ".yaml" or ".yml" or ".xml" or ".properties" or ".conf" or ".config" + or ".env" or ".ini" or ".toml" or ".cfg" + or ".js" or ".ts" or ".jsx" or ".tsx" or ".mjs" or ".cjs" + or ".py" or ".rb" or ".php" or ".go" or ".java" or ".cs" or ".rs" or ".swift" or ".kt" + or ".sh" or ".bash" or ".zsh" or ".ps1" or ".bat" or ".cmd" + or ".sql" or ".graphql" or ".gql" + or ".tf" or ".tfvars" or ".hcl" + or ".dockerfile" or ".dockerignore" + or ".gitignore" or ".npmrc" or ".yarnrc" or ".pypirc" + or ".pem" or ".key" or ".crt" or ".cer" + or ".md" or ".txt" or ".log" + || string.IsNullOrEmpty(ext); + } + + private static string? ResolveRootfsPath(IReadOnlyDictionary metadata) + { + if (metadata is null) + { + return null; + } + + foreach (var key in RootFsMetadataKeys) + { + if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + + return null; + } +} + +/// +/// Report of secrets analysis for a scan job. +/// +public sealed record SecretsAnalysisReport +{ + public required string JobId { get; init; } + public required string ScanId { get; init; } + public required ImmutableArray Findings { get; init; } + public required int FilesScanned { get; init; } + public required string RulesetVersion { get; init; } + public required DateTimeOffset AnalyzedAtUtc { get; init; } + public required double ElapsedMilliseconds { get; init; } +} diff --git a/src/Scanner/StellaOps.Scanner.Worker/Program.cs b/src/Scanner/StellaOps.Scanner.Worker/Program.cs index d9d6353e5..fc48a3314 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/Program.cs +++ b/src/Scanner/StellaOps.Scanner.Worker/Program.cs @@ -26,7 +26,9 @@ using StellaOps.Scanner.Worker.Hosting; using StellaOps.Scanner.Worker.Options; using StellaOps.Scanner.Worker.Processing; using StellaOps.Scanner.Worker.Processing.Entropy; +using StellaOps.Scanner.Worker.Processing.Secrets; using StellaOps.Scanner.Worker.Determinism; +using StellaOps.Scanner.Analyzers.Secrets; using StellaOps.Scanner.Worker.Extensions; using StellaOps.Scanner.Worker.Processing.Surface; using StellaOps.Scanner.Storage.Extensions; @@ -167,6 +169,18 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); +// Secrets Leak Detection (Sprint: SPRINT_20251229_046_BE) +if (workerOptions.Secrets.Enabled) +{ + builder.Services.AddSecretsAnalyzer(options => + { + options.RulesetPath = workerOptions.Secrets.RulesetPath; + options.EnableEntropyDetection = workerOptions.Secrets.EnableEntropyDetection; + options.EntropyThreshold = workerOptions.Secrets.EntropyThreshold; + }); + builder.Services.AddSingleton(); +} + // Proof of Exposure (Sprint: SPRINT_3500_0001_0001_proof_of_exposure_mvp) builder.Services.AddOptions() .BindConfiguration("PoE") diff --git a/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj b/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj index 54cc3770d..63f044bbe 100644 --- a/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj +++ b/src/Scanner/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj @@ -27,6 +27,7 @@ + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj index e69de29bb..8cc6a515a 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj @@ -0,0 +1,21 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AGENTS.md b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AGENTS.md new file mode 100644 index 000000000..4cfa263d5 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AGENTS.md @@ -0,0 +1,128 @@ +# Scanner Secrets Analyzer Guild Charter + +## Mission + +Detect accidentally committed secrets in container layers during scans using deterministic, DSSE-signed rule bundles. Ensure findings are reproducible, masked before output, and integrated with the Policy Engine for policy-driven decisions. + +## Scope + +- Secret detection plugin implementing `ILayerAnalyzer` +- Regex and entropy-based detection strategies +- Rule bundle loading, verification, and execution +- Payload masking engine +- Evidence emission (`secret.leak`) for policy integration +- Integration with Scanner Worker pipeline + +## Required Reading + +- `docs/modules/scanner/operations/secret-leak-detection.md` - Target specification +- `docs/modules/scanner/design/surface-secrets.md` - Credential delivery (different from leak detection) +- `docs/modules/scanner/architecture.md` - Scanner module architecture +- `docs/modules/policy/secret-leak-detection-readiness.md` - Policy integration requirements +- `docs/implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md` - Implementation sprint +- `docs/implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md` - Bundle infrastructure sprint +- CLAUDE.md Section 8 (Code Quality & Determinism Rules) + +## Working Agreement + +1. **Status synchronisation**: Update task state in sprint file and local `TASKS.md` when starting or completing work. + +2. **Determinism**: + - Sort rules by ID for deterministic execution order + - Use `CultureInfo.InvariantCulture` for all parsing + - Inject `TimeProvider` for timestamps + - Same inputs must produce same outputs + +3. **Security posture**: + - NEVER log secret payloads + - Apply masking BEFORE any output or persistence + - Verify bundle signatures on load + - Enforce feature flag for gradual rollout + +4. **Testing requirements**: + - Unit tests for all detectors, masking, and rule loading + - Integration tests with Scanner Worker + - Golden fixture tests for determinism verification + - Security tests ensuring secrets are not leaked + +5. **Offline readiness**: + - Support local bundle verification without network + - Document Attestor mirror configuration + - Ensure bundles ship with Offline Kit + +## Key Interfaces + +```csharp +// Detection interface +public interface ISecretDetector +{ + string DetectorId { get; } + ValueTask> DetectAsync( + ReadOnlyMemory content, + string filePath, + SecretRule rule, + CancellationToken ct = default); +} + +// Masking interface +public interface IPayloadMasker +{ + string Mask(ReadOnlySpan payload, string? hint = null); +} + +// Bundle verification +public interface IBundleVerifier +{ + Task VerifyAsync( + string bundleDirectory, + VerificationOptions options, + CancellationToken ct = default); +} +``` + +## Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `scanner.secret.finding_total` | Counter | Total findings by tenant, ruleId, severity | +| `scanner.secret.scan_duration_seconds` | Histogram | Detection time per scan | +| `scanner.secret.rules_loaded` | Gauge | Number of active rules | + +## Directory Structure + +``` +StellaOps.Scanner.Analyzers.Secrets/ +├── AGENTS.md # This file +├── StellaOps.Scanner.Analyzers.Secrets.csproj +├── Detectors/ +│ ├── ISecretDetector.cs +│ ├── RegexDetector.cs +│ ├── EntropyDetector.cs +│ └── CompositeSecretDetector.cs +├── Rules/ +│ ├── SecretRule.cs +│ ├── SecretRuleset.cs +│ └── RulesetLoader.cs +├── Bundles/ +│ ├── BundleBuilder.cs +│ ├── BundleVerifier.cs +│ └── Schemas/ +├── Masking/ +│ ├── IPayloadMasker.cs +│ └── PayloadMasker.cs +├── Evidence/ +│ ├── SecretLeakEvidence.cs +│ └── SecretFinding.cs +├── SecretsAnalyzer.cs +├── SecretsAnalyzerHost.cs +├── SecretsAnalyzerOptions.cs +└── ServiceCollectionExtensions.cs +``` + +## Implementation Status + +See sprint files for current implementation status: +- SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md +- SPRINT_20260104_003_SCANNER_secret_rule_bundles.md +- SPRINT_20260104_004_POLICY_secret_dsl_integration.md +- SPRINT_20260104_005_AIRGAP_secret_offline_kit.md diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AssemblyInfo.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AssemblyInfo.cs new file mode 100644 index 000000000..9555527d1 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.Secrets.Tests")] diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleBuilder.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleBuilder.cs new file mode 100644 index 000000000..0e370bbef --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleBuilder.cs @@ -0,0 +1,345 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Analyzers.Secrets.Bundles; + +/// +/// Builds secrets detection rule bundles from individual rule files. +/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles +/// +public interface IBundleBuilder +{ + /// + /// Creates a bundle from individual rule files. + /// + Task BuildAsync( + BundleBuildOptions options, + CancellationToken ct = default); +} + +/// +/// Options for bundle creation. +/// +public sealed record BundleBuildOptions +{ + /// + /// Directory where the bundle will be written. + /// + public required string OutputDirectory { get; init; } + + /// + /// Bundle identifier (e.g., "secrets.ruleset"). + /// + public required string BundleId { get; init; } + + /// + /// Bundle version (e.g., "2026.01"). + /// + public required string Version { get; init; } + + /// + /// Paths to individual rule JSON files to include. + /// + public required IReadOnlyList RuleFiles { get; init; } + + /// + /// Optional description for the bundle. + /// + public string Description { get; init; } = "StellaOps Secret Detection Rules"; + + /// + /// Time provider for deterministic timestamps. + /// + public TimeProvider? TimeProvider { get; init; } + + /// + /// Whether to validate rules during build. + /// + public bool ValidateRules { get; init; } = true; + + /// + /// Whether to fail on validation warnings. + /// + public bool FailOnWarnings { get; init; } = false; +} + +/// +/// Result of bundle creation. +/// +public sealed record BundleArtifact +{ + /// + /// Path to the manifest file. + /// + public required string ManifestPath { get; init; } + + /// + /// Path to the rules JSONL file. + /// + public required string RulesPath { get; init; } + + /// + /// SHA-256 hash of the rules file (lowercase hex). + /// + public required string RulesSha256 { get; init; } + + /// + /// Total number of rules in the bundle. + /// + public required int TotalRules { get; init; } + + /// + /// Number of enabled rules in the bundle. + /// + public required int EnabledRules { get; init; } + + /// + /// The generated manifest. + /// + public required BundleManifest Manifest { get; init; } +} + +/// +/// Default implementation of bundle builder. +/// +public sealed class BundleBuilder : IBundleBuilder +{ + private static readonly JsonSerializerOptions ManifestSerializerOptions = new() + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + private static readonly JsonSerializerOptions RuleSerializerOptions = new() + { + WriteIndented = false, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingDefault + }; + + private static readonly JsonSerializerOptions RuleReaderOptions = new() + { + PropertyNameCaseInsensitive = true, + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip + }; + + private readonly IRuleValidator _validator; + private readonly ILogger _logger; + + public BundleBuilder(IRuleValidator validator, ILogger logger) + { + _validator = validator ?? throw new ArgumentNullException(nameof(validator)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task BuildAsync( + BundleBuildOptions options, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(options); + + var timeProvider = options.TimeProvider ?? TimeProvider.System; + var createdAt = timeProvider.GetUtcNow(); + + _logger.LogInformation( + "Building bundle {BundleId} v{Version} from {FileCount} rule files", + options.BundleId, + options.Version, + options.RuleFiles.Count); + + // Load and validate rules + var rules = new List(); + var validationErrors = new List(); + var validationWarnings = new List(); + + foreach (var ruleFile in options.RuleFiles) + { + ct.ThrowIfCancellationRequested(); + + if (!File.Exists(ruleFile)) + { + validationErrors.Add($"Rule file not found: {ruleFile}"); + continue; + } + + try + { + var json = await File.ReadAllTextAsync(ruleFile, ct).ConfigureAwait(false); + var rule = JsonSerializer.Deserialize(json, RuleReaderOptions); + + if (rule is null) + { + validationErrors.Add($"Failed to deserialize rule from {ruleFile}"); + continue; + } + + if (options.ValidateRules) + { + var validation = _validator.Validate(rule); + if (!validation.IsValid) + { + foreach (var error in validation.Errors) + { + validationErrors.Add($"{ruleFile}: {error}"); + } + continue; + } + + foreach (var warning in validation.Warnings) + { + validationWarnings.Add($"{ruleFile}: {warning}"); + } + } + + rules.Add(rule); + } + catch (JsonException ex) + { + validationErrors.Add($"JSON parse error in {ruleFile}: {ex.Message}"); + } + } + + // Handle warnings + if (validationWarnings.Count > 0) + { + _logger.LogWarning( + "Bundle build has {WarningCount} warnings: {Warnings}", + validationWarnings.Count, + string.Join("; ", validationWarnings.Take(5))); + + if (options.FailOnWarnings) + { + throw new InvalidOperationException( + $"Bundle build failed due to warnings: {string.Join("; ", validationWarnings)}"); + } + } + + // Handle errors + if (validationErrors.Count > 0) + { + throw new InvalidOperationException( + $"Bundle build failed with {validationErrors.Count} errors: {string.Join("; ", validationErrors)}"); + } + + if (rules.Count == 0) + { + throw new InvalidOperationException("No valid rules found to include in bundle."); + } + + // Sort rules by ID for deterministic output + rules.Sort((a, b) => string.Compare(a.Id, b.Id, StringComparison.Ordinal)); + + // Ensure output directory exists + Directory.CreateDirectory(options.OutputDirectory); + + // Write rules JSONL file + var rulesPath = Path.Combine(options.OutputDirectory, "secrets.ruleset.rules.jsonl"); + await WriteRulesJsonlAsync(rulesPath, rules, ct).ConfigureAwait(false); + + // Compute SHA-256 of rules file + var rulesSha256 = await ComputeFileSha256Async(rulesPath, ct).ConfigureAwait(false); + + // Build manifest + var manifest = new BundleManifest + { + SchemaVersion = "1.0", + Id = options.BundleId, + Version = options.Version, + CreatedAt = createdAt, + Description = options.Description, + Rules = rules.Select(r => new BundleRuleSummary + { + Id = r.Id, + Version = r.Version, + Severity = r.Severity.ToString().ToLowerInvariant(), + Enabled = r.Enabled + }).ToImmutableArray(), + Integrity = new BundleIntegrity + { + RulesFile = "secrets.ruleset.rules.jsonl", + RulesSha256 = rulesSha256, + TotalRules = rules.Count, + EnabledRules = rules.Count(r => r.Enabled) + } + }; + + // Write manifest + var manifestPath = Path.Combine(options.OutputDirectory, "secrets.ruleset.manifest.json"); + await WriteManifestAsync(manifestPath, manifest, ct).ConfigureAwait(false); + + _logger.LogInformation( + "Bundle {BundleId} v{Version} created with {RuleCount} rules ({EnabledCount} enabled)", + options.BundleId, + options.Version, + rules.Count, + rules.Count(r => r.Enabled)); + + return new BundleArtifact + { + ManifestPath = manifestPath, + RulesPath = rulesPath, + RulesSha256 = rulesSha256, + TotalRules = rules.Count, + EnabledRules = rules.Count(r => r.Enabled), + Manifest = manifest + }; + } + + private static async Task WriteRulesJsonlAsync( + string path, + IReadOnlyList rules, + CancellationToken ct) + { + await using var stream = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + useAsync: true); + + await using var writer = new StreamWriter(stream, Encoding.UTF8, leaveOpen: true); + + foreach (var rule in rules) + { + ct.ThrowIfCancellationRequested(); + var json = JsonSerializer.Serialize(rule, RuleSerializerOptions); + await writer.WriteLineAsync(json).ConfigureAwait(false); + } + } + + private static async Task WriteManifestAsync( + string path, + BundleManifest manifest, + CancellationToken ct) + { + var json = JsonSerializer.Serialize(manifest, ManifestSerializerOptions); + await File.WriteAllTextAsync(path, json, Encoding.UTF8, ct).ConfigureAwait(false); + } + + private static async Task ComputeFileSha256Async(string path, CancellationToken ct) + { + await using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 4096, + useAsync: true); + + var hash = await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleManifest.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleManifest.cs new file mode 100644 index 000000000..45dacef81 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleManifest.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Analyzers.Secrets.Bundles; + +/// +/// Represents the manifest of a secrets detection rule bundle. +/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles +/// +public sealed record BundleManifest +{ + /// + /// Schema version identifier. + /// + [JsonPropertyName("schemaVersion")] + public string SchemaVersion { get; init; } = "1.0"; + + /// + /// Unique identifier for the bundle (e.g., "secrets.ruleset"). + /// + [JsonPropertyName("id")] + public required string Id { get; init; } + + /// + /// Bundle version using CalVer (e.g., "2026.01"). + /// + [JsonPropertyName("version")] + public required string Version { get; init; } + + /// + /// UTC timestamp when the bundle was created. + /// + [JsonPropertyName("createdAt")] + public required DateTimeOffset CreatedAt { get; init; } + + /// + /// Human-readable description of the bundle. + /// + [JsonPropertyName("description")] + public string Description { get; init; } = string.Empty; + + /// + /// Summary of rules included in the bundle. + /// + [JsonPropertyName("rules")] + public ImmutableArray Rules { get; init; } = []; + + /// + /// Integrity information for the bundle. + /// + [JsonPropertyName("integrity")] + public required BundleIntegrity Integrity { get; init; } + + /// + /// Signature information for the bundle. + /// + [JsonPropertyName("signatures")] + public BundleSignatures? Signatures { get; init; } +} + +/// +/// Summary of a rule included in the bundle manifest. +/// +public sealed record BundleRuleSummary +{ + /// + /// Unique rule identifier. + /// + [JsonPropertyName("id")] + public required string Id { get; init; } + + /// + /// Rule version (SemVer). + /// + [JsonPropertyName("version")] + public required string Version { get; init; } + + /// + /// Rule severity level. + /// + [JsonPropertyName("severity")] + public required string Severity { get; init; } + + /// + /// Whether the rule is enabled by default. + /// + [JsonPropertyName("enabled")] + public bool Enabled { get; init; } = true; +} + +/// +/// Integrity information for bundle verification. +/// +public sealed record BundleIntegrity +{ + /// + /// Name of the rules file within the bundle. + /// + [JsonPropertyName("rulesFile")] + public string RulesFile { get; init; } = "secrets.ruleset.rules.jsonl"; + + /// + /// SHA-256 hash of the rules file (lowercase hex). + /// + [JsonPropertyName("rulesSha256")] + public required string RulesSha256 { get; init; } + + /// + /// Total number of rules in the bundle. + /// + [JsonPropertyName("totalRules")] + public required int TotalRules { get; init; } + + /// + /// Number of enabled rules in the bundle. + /// + [JsonPropertyName("enabledRules")] + public required int EnabledRules { get; init; } +} + +/// +/// Signature references for the bundle. +/// +public sealed record BundleSignatures +{ + /// + /// Path to the DSSE envelope file within the bundle. + /// + [JsonPropertyName("dsseEnvelope")] + public string DsseEnvelope { get; init; } = "secrets.ruleset.dsse.json"; + + /// + /// Key ID used for signing (for informational purposes). + /// + [JsonPropertyName("keyId")] + public string? KeyId { get; init; } + + /// + /// UTC timestamp when the bundle was signed. + /// + [JsonPropertyName("signedAt")] + public DateTimeOffset? SignedAt { get; init; } + + /// + /// Rekor transparency log entry ID (if applicable). + /// + [JsonPropertyName("rekorLogId")] + public string? RekorLogId { get; init; } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleSigner.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleSigner.cs new file mode 100644 index 000000000..fc3bcbf75 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleSigner.cs @@ -0,0 +1,349 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Analyzers.Secrets.Bundles; + +/// +/// Signs secrets detection rule bundles using DSSE envelopes. +/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles +/// +public interface IBundleSigner +{ + /// + /// Signs a bundle artifact producing a DSSE envelope. + /// + Task SignAsync( + BundleArtifact artifact, + BundleSigningOptions options, + CancellationToken ct = default); +} + +/// +/// Options for bundle signing. +/// +public sealed record BundleSigningOptions +{ + /// + /// Key identifier for the signature. + /// + public required string KeyId { get; init; } + + /// + /// Signing algorithm (e.g., "HMAC-SHA256", "ES256"). + /// + public string Algorithm { get; init; } = "HMAC-SHA256"; + + /// + /// Shared secret for HMAC signing (base64 or hex encoded). + /// Required for HMAC-SHA256 algorithm. + /// + public string? SharedSecret { get; init; } + + /// + /// Path to file containing the shared secret. + /// + public string? SharedSecretFile { get; init; } + + /// + /// Payload type for the DSSE envelope. + /// + public string PayloadType { get; init; } = "application/vnd.stellaops.secrets-ruleset+json"; + + /// + /// Time provider for deterministic timestamps. + /// + public TimeProvider? TimeProvider { get; init; } +} + +/// +/// Result of bundle signing. +/// +public sealed record BundleSigningResult +{ + /// + /// Path to the generated DSSE envelope file. + /// + public required string EnvelopePath { get; init; } + + /// + /// The generated DSSE envelope. + /// + public required DsseEnvelope Envelope { get; init; } + + /// + /// Updated manifest with signature information. + /// + public required BundleManifest UpdatedManifest { get; init; } +} + +/// +/// DSSE envelope structure for bundle signatures. +/// +public sealed record DsseEnvelope +{ + /// + /// Base64url-encoded payload. + /// + public required string Payload { get; init; } + + /// + /// Payload type URI. + /// + public required string PayloadType { get; init; } + + /// + /// Signatures over the PAE. + /// + public required ImmutableArray Signatures { get; init; } +} + +/// +/// A signature within a DSSE envelope. +/// +public sealed record DsseSignature +{ + /// + /// Base64url-encoded signature bytes. + /// + public required string Sig { get; init; } + + /// + /// Key identifier. + /// + public string? KeyId { get; init; } +} + +/// +/// Default implementation of bundle signing using HMAC-SHA256. +/// +public sealed class BundleSigner : IBundleSigner +{ + private const string DssePrefix = "DSSEv1"; + + private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new() + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + private readonly ILogger _logger; + + public BundleSigner(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task SignAsync( + BundleArtifact artifact, + BundleSigningOptions options, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(artifact); + ArgumentNullException.ThrowIfNull(options); + + var timeProvider = options.TimeProvider ?? TimeProvider.System; + var signedAt = timeProvider.GetUtcNow(); + + _logger.LogInformation( + "Signing bundle {BundleId} v{Version} with key {KeyId}", + artifact.Manifest.Id, + artifact.Manifest.Version, + options.KeyId); + + // Read manifest as payload + var manifestJson = await File.ReadAllBytesAsync(artifact.ManifestPath, ct).ConfigureAwait(false); + + // Encode payload as base64url + var payloadBase64 = ToBase64Url(manifestJson); + + // Build PAE (Pre-Authentication Encoding) + var pae = BuildPae(options.PayloadType, manifestJson); + + // Sign the PAE + var signature = await SignPaeAsync(pae, options, ct).ConfigureAwait(false); + var signatureBase64 = ToBase64Url(signature); + + // Build DSSE envelope + var envelope = new DsseEnvelope + { + Payload = payloadBase64, + PayloadType = options.PayloadType, + Signatures = + [ + new DsseSignature + { + Sig = signatureBase64, + KeyId = options.KeyId + } + ] + }; + + // Write envelope to file + var bundleDir = Path.GetDirectoryName(artifact.ManifestPath)!; + var envelopePath = Path.Combine(bundleDir, "secrets.ruleset.dsse.json"); + var envelopeJson = JsonSerializer.Serialize(envelope, EnvelopeSerializerOptions); + await File.WriteAllTextAsync(envelopePath, envelopeJson, Encoding.UTF8, ct).ConfigureAwait(false); + + // Update manifest with signature info + var updatedManifest = artifact.Manifest with + { + Signatures = new BundleSignatures + { + DsseEnvelope = "secrets.ruleset.dsse.json", + KeyId = options.KeyId, + SignedAt = signedAt + } + }; + + // Rewrite manifest with signature info + var updatedManifestJson = JsonSerializer.Serialize(updatedManifest, EnvelopeSerializerOptions); + await File.WriteAllTextAsync(artifact.ManifestPath, updatedManifestJson, Encoding.UTF8, ct).ConfigureAwait(false); + + _logger.LogInformation( + "Bundle signed successfully. Envelope: {EnvelopePath}", + envelopePath); + + return new BundleSigningResult + { + EnvelopePath = envelopePath, + Envelope = envelope, + UpdatedManifest = updatedManifest + }; + } + + private async Task SignPaeAsync( + byte[] pae, + BundleSigningOptions options, + CancellationToken ct) + { + if (!options.Algorithm.Equals("HMAC-SHA256", StringComparison.OrdinalIgnoreCase)) + { + throw new NotSupportedException($"Algorithm '{options.Algorithm}' is not supported. Use HMAC-SHA256."); + } + + var secret = await LoadSecretAsync(options, ct).ConfigureAwait(false); + if (secret is null || secret.Length == 0) + { + throw new InvalidOperationException("Shared secret is required for HMAC-SHA256 signing."); + } + + using var hmac = new HMACSHA256(secret); + return hmac.ComputeHash(pae); + } + + private static async Task LoadSecretAsync(BundleSigningOptions options, CancellationToken ct) + { + // Try file first + if (!string.IsNullOrWhiteSpace(options.SharedSecretFile) && File.Exists(options.SharedSecretFile)) + { + var content = (await File.ReadAllTextAsync(options.SharedSecretFile, ct).ConfigureAwait(false)).Trim(); + return DecodeSecret(content); + } + + // Then inline secret + if (!string.IsNullOrWhiteSpace(options.SharedSecret)) + { + return DecodeSecret(options.SharedSecret); + } + + return null; + } + + private static byte[] DecodeSecret(string value) + { + // Try base64 first + try + { + return Convert.FromBase64String(value); + } + catch (FormatException) + { + // Not base64 + } + + // Try hex + if (value.Length % 2 == 0 && IsHexString(value)) + { + return Convert.FromHexString(value); + } + + // Treat as raw UTF-8 + return Encoding.UTF8.GetBytes(value); + } + + private static bool IsHexString(string value) + { + foreach (var c in value) + { + if (!char.IsAsciiHexDigit(c)) + return false; + } + return true; + } + + /// + /// Builds DSSE v1 Pre-Authentication Encoding. + /// Format: "DSSEv1" SP LEN(type) SP type SP LEN(payload) SP payload + /// + private static byte[] BuildPae(string payloadType, byte[] payload) + { + var typeBytes = Encoding.UTF8.GetBytes(payloadType); + var typeLenStr = typeBytes.Length.ToString(System.Globalization.CultureInfo.InvariantCulture); + var payloadLenStr = payload.Length.ToString(System.Globalization.CultureInfo.InvariantCulture); + + // Calculate total size + var prefixBytes = Encoding.UTF8.GetBytes(DssePrefix); + var typeLenBytes = Encoding.UTF8.GetBytes(typeLenStr); + var payloadLenBytes = Encoding.UTF8.GetBytes(payloadLenStr); + + var totalSize = prefixBytes.Length + 1 // prefix + SP + + typeLenBytes.Length + 1 // type len + SP + + typeBytes.Length + 1 // type + SP + + payloadLenBytes.Length + 1 // payload len + SP + + payload.Length; + + var pae = new byte[totalSize]; + var offset = 0; + + // DSSEv1 + Buffer.BlockCopy(prefixBytes, 0, pae, offset, prefixBytes.Length); + offset += prefixBytes.Length; + pae[offset++] = 0x20; // SP + + // type length + Buffer.BlockCopy(typeLenBytes, 0, pae, offset, typeLenBytes.Length); + offset += typeLenBytes.Length; + pae[offset++] = 0x20; // SP + + // type + Buffer.BlockCopy(typeBytes, 0, pae, offset, typeBytes.Length); + offset += typeBytes.Length; + pae[offset++] = 0x20; // SP + + // payload length + Buffer.BlockCopy(payloadLenBytes, 0, pae, offset, payloadLenBytes.Length); + offset += payloadLenBytes.Length; + pae[offset++] = 0x20; // SP + + // payload + Buffer.BlockCopy(payload, 0, pae, offset, payload.Length); + + return pae; + } + + private static string ToBase64Url(byte[] data) + { + return Convert.ToBase64String(data) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleVerifier.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleVerifier.cs new file mode 100644 index 000000000..73ccb22be --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/BundleVerifier.cs @@ -0,0 +1,527 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Analyzers.Secrets.Bundles; + +/// +/// Verifies secrets detection rule bundle signatures and integrity. +/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles +/// +public interface IBundleVerifier +{ + /// + /// Verifies a bundle's DSSE signature and integrity. + /// + Task VerifyAsync( + string bundleDirectory, + BundleVerificationOptions options, + CancellationToken ct = default); +} + +/// +/// Options for bundle verification. +/// +public sealed record BundleVerificationOptions +{ + /// + /// URL of the attestor service for online verification. + /// + public string? AttestorUrl { get; init; } + + /// + /// Whether to require Rekor transparency log proof. + /// + public bool RequireRekorProof { get; init; } = false; + + /// + /// List of trusted key IDs. If empty, any key is accepted. + /// + public IReadOnlyList? TrustedKeyIds { get; init; } + + /// + /// Shared secret for HMAC verification (base64 or hex encoded). + /// + public string? SharedSecret { get; init; } + + /// + /// Path to file containing the shared secret. + /// + public string? SharedSecretFile { get; init; } + + /// + /// Whether to verify file integrity (SHA-256). + /// + public bool VerifyIntegrity { get; init; } = true; + + /// + /// Whether to skip signature verification (integrity only). + /// + public bool SkipSignatureVerification { get; init; } = false; +} + +/// +/// Result of bundle verification. +/// +public sealed record BundleVerificationResult +{ + /// + /// Whether the bundle is valid. + /// + public required bool IsValid { get; init; } + + /// + /// Bundle version that was verified. + /// + public string? BundleVersion { get; init; } + + /// + /// Bundle ID that was verified. + /// + public string? BundleId { get; init; } + + /// + /// UTC timestamp when the bundle was signed. + /// + public DateTimeOffset? SignedAt { get; init; } + + /// + /// Key ID that signed the bundle. + /// + public string? SignerKeyId { get; init; } + + /// + /// Rekor transparency log entry ID (if available). + /// + public string? RekorLogId { get; init; } + + /// + /// Total number of rules in the bundle. + /// + public int? RuleCount { get; init; } + + /// + /// Validation errors encountered. + /// + public ImmutableArray ValidationErrors { get; init; } = []; + + /// + /// Validation warnings (non-fatal). + /// + public ImmutableArray ValidationWarnings { get; init; } = []; + + public static BundleVerificationResult Success(BundleManifest manifest, string? keyId) => new() + { + IsValid = true, + BundleId = manifest.Id, + BundleVersion = manifest.Version, + SignedAt = manifest.Signatures?.SignedAt, + SignerKeyId = keyId ?? manifest.Signatures?.KeyId, + RekorLogId = manifest.Signatures?.RekorLogId, + RuleCount = manifest.Integrity.TotalRules + }; + + public static BundleVerificationResult Failure(params string[] errors) => new() + { + IsValid = false, + ValidationErrors = [.. errors] + }; + + public static BundleVerificationResult Failure(IEnumerable errors) => new() + { + IsValid = false, + ValidationErrors = [.. errors] + }; +} + +/// +/// Default implementation of bundle verification. +/// +public sealed class BundleVerifier : IBundleVerifier +{ + private const string DssePrefix = "DSSEv1"; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + AllowTrailingCommas = true + }; + + private readonly ILogger _logger; + + public BundleVerifier(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task VerifyAsync( + string bundleDirectory, + BundleVerificationOptions options, + CancellationToken ct = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(bundleDirectory); + ArgumentNullException.ThrowIfNull(options); + + var errors = new List(); + var warnings = new List(); + + _logger.LogDebug("Verifying bundle at {BundleDir}", bundleDirectory); + + // Check directory exists + if (!Directory.Exists(bundleDirectory)) + { + return BundleVerificationResult.Failure($"Bundle directory not found: {bundleDirectory}"); + } + + // Load manifest + var manifestPath = Path.Combine(bundleDirectory, "secrets.ruleset.manifest.json"); + if (!File.Exists(manifestPath)) + { + return BundleVerificationResult.Failure($"Manifest not found: {manifestPath}"); + } + + BundleManifest manifest; + try + { + var manifestJson = await File.ReadAllTextAsync(manifestPath, ct).ConfigureAwait(false); + manifest = JsonSerializer.Deserialize(manifestJson, JsonOptions)!; + if (manifest is null) + { + return BundleVerificationResult.Failure("Failed to parse manifest."); + } + } + catch (JsonException ex) + { + return BundleVerificationResult.Failure($"Invalid manifest JSON: {ex.Message}"); + } + + _logger.LogDebug( + "Loaded manifest: {BundleId} v{Version} with {RuleCount} rules", + manifest.Id, + manifest.Version, + manifest.Integrity.TotalRules); + + // Verify file integrity + if (options.VerifyIntegrity) + { + var rulesPath = Path.Combine(bundleDirectory, manifest.Integrity.RulesFile); + if (!File.Exists(rulesPath)) + { + errors.Add($"Rules file not found: {rulesPath}"); + } + else + { + var actualHash = await ComputeFileSha256Async(rulesPath, ct).ConfigureAwait(false); + if (!string.Equals(actualHash, manifest.Integrity.RulesSha256, StringComparison.OrdinalIgnoreCase)) + { + errors.Add($"Rules file integrity check failed. Expected: {manifest.Integrity.RulesSha256}, Actual: {actualHash}"); + } + else + { + _logger.LogDebug("Rules file integrity verified."); + } + } + } + + // Verify signature + if (!options.SkipSignatureVerification) + { + if (manifest.Signatures is null) + { + errors.Add("Bundle is not signed."); + } + else + { + var envelopePath = Path.Combine(bundleDirectory, manifest.Signatures.DsseEnvelope); + if (!File.Exists(envelopePath)) + { + errors.Add($"DSSE envelope not found: {envelopePath}"); + } + else + { + var signatureResult = await VerifySignatureAsync( + manifestPath, + envelopePath, + options, + ct).ConfigureAwait(false); + + if (!signatureResult.IsValid) + { + errors.AddRange(signatureResult.Errors); + } + else + { + // Check trusted key IDs + if (options.TrustedKeyIds is { Count: > 0 } trustedKeys) + { + if (signatureResult.KeyId is null || !trustedKeys.Contains(signatureResult.KeyId)) + { + errors.Add($"Signature key '{signatureResult.KeyId}' is not in the trusted keys list."); + } + } + + _logger.LogDebug("Signature verified with key: {KeyId}", signatureResult.KeyId); + } + } + } + } + + // Check Rekor requirement + if (options.RequireRekorProof) + { + if (manifest.Signatures?.RekorLogId is null) + { + errors.Add("Rekor transparency log proof is required but not present."); + } + else + { + // TODO: Implement Rekor verification via Attestor client + warnings.Add("Rekor verification not yet implemented; proof present but not verified."); + } + } + + if (errors.Count > 0) + { + _logger.LogWarning( + "Bundle verification failed for {BundleId} v{Version}: {Errors}", + manifest.Id, + manifest.Version, + string.Join("; ", errors)); + + return new BundleVerificationResult + { + IsValid = false, + BundleId = manifest.Id, + BundleVersion = manifest.Version, + ValidationErrors = [.. errors], + ValidationWarnings = [.. warnings] + }; + } + + _logger.LogInformation( + "Bundle verified: {BundleId} v{Version} ({RuleCount} rules)", + manifest.Id, + manifest.Version, + manifest.Integrity.TotalRules); + + return new BundleVerificationResult + { + IsValid = true, + BundleId = manifest.Id, + BundleVersion = manifest.Version, + SignedAt = manifest.Signatures?.SignedAt, + SignerKeyId = manifest.Signatures?.KeyId, + RekorLogId = manifest.Signatures?.RekorLogId, + RuleCount = manifest.Integrity.TotalRules, + ValidationWarnings = [.. warnings] + }; + } + + private async Task VerifySignatureAsync( + string manifestPath, + string envelopePath, + BundleVerificationOptions options, + CancellationToken ct) + { + try + { + // Load envelope + var envelopeJson = await File.ReadAllTextAsync(envelopePath, ct).ConfigureAwait(false); + var envelope = JsonSerializer.Deserialize(envelopeJson, JsonOptions); + + if (envelope is null || envelope.Signatures.IsDefaultOrEmpty) + { + return SignatureVerificationResult.Failure("Invalid or empty DSSE envelope."); + } + + // Decode payload - this is the original manifest (before signature was added) + var payloadBytes = FromBase64Url(envelope.Payload); + var payloadManifest = JsonSerializer.Deserialize(payloadBytes, JsonOptions); + + if (payloadManifest is null) + { + return SignatureVerificationResult.Failure("Failed to parse envelope payload as manifest."); + } + + // Load current manifest and verify it matches the signed version (ignoring the Signatures field + // which was added after signing) + var manifestJson = await File.ReadAllTextAsync(manifestPath, ct).ConfigureAwait(false); + var currentManifest = JsonSerializer.Deserialize(manifestJson, JsonOptions); + + if (currentManifest is null) + { + return SignatureVerificationResult.Failure("Failed to parse current manifest."); + } + + // Compare all fields except Signatures (which is added after signing) + if (!ManifestsMatchIgnoringSignatures(payloadManifest, currentManifest)) + { + return SignatureVerificationResult.Failure("Envelope payload does not match manifest content."); + } + + // Build PAE + var pae = BuildPae(envelope.PayloadType, payloadBytes); + + // Verify each signature (at least one must be valid) + var secret = await LoadSecretAsync(options, ct).ConfigureAwait(false); + if (secret is null || secret.Length == 0) + { + return SignatureVerificationResult.Failure("Shared secret is required for signature verification."); + } + + foreach (var sig in envelope.Signatures) + { + var signatureBytes = FromBase64Url(sig.Sig); + + using var hmac = new HMACSHA256(secret); + var expectedSignature = hmac.ComputeHash(pae); + + if (CryptographicOperations.FixedTimeEquals(expectedSignature, signatureBytes)) + { + return SignatureVerificationResult.Success(sig.KeyId); + } + } + + return SignatureVerificationResult.Failure("Signature verification failed."); + } + catch (Exception ex) + { + return SignatureVerificationResult.Failure($"Signature verification error: {ex.Message}"); + } + } + + private static bool ManifestsMatchIgnoringSignatures(BundleManifest a, BundleManifest b) + { + // Compare all fields except Signatures + return a.SchemaVersion == b.SchemaVersion + && a.Id == b.Id + && a.Version == b.Version + && a.CreatedAt == b.CreatedAt + && a.Description == b.Description + && a.Integrity.RulesFile == b.Integrity.RulesFile + && a.Integrity.RulesSha256 == b.Integrity.RulesSha256 + && a.Integrity.TotalRules == b.Integrity.TotalRules + && a.Integrity.EnabledRules == b.Integrity.EnabledRules; + } + + private static async Task LoadSecretAsync(BundleVerificationOptions options, CancellationToken ct) + { + // Try file first + if (!string.IsNullOrWhiteSpace(options.SharedSecretFile) && File.Exists(options.SharedSecretFile)) + { + var content = (await File.ReadAllTextAsync(options.SharedSecretFile, ct).ConfigureAwait(false)).Trim(); + return DecodeSecret(content); + } + + // Then inline secret + if (!string.IsNullOrWhiteSpace(options.SharedSecret)) + { + return DecodeSecret(options.SharedSecret); + } + + return null; + } + + private static byte[] DecodeSecret(string value) + { + // Try base64 first + try + { + return Convert.FromBase64String(value); + } + catch (FormatException) + { + // Not base64 + } + + // Try hex + if (value.Length % 2 == 0 && IsHexString(value)) + { + return Convert.FromHexString(value); + } + + // Treat as raw UTF-8 + return Encoding.UTF8.GetBytes(value); + } + + private static bool IsHexString(string value) + { + foreach (var c in value) + { + if (!char.IsAsciiHexDigit(c)) + return false; + } + return true; + } + + private static byte[] BuildPae(string payloadType, byte[] payload) + { + var typeBytes = Encoding.UTF8.GetBytes(payloadType); + var typeLenStr = typeBytes.Length.ToString(System.Globalization.CultureInfo.InvariantCulture); + var payloadLenStr = payload.Length.ToString(System.Globalization.CultureInfo.InvariantCulture); + + var prefixBytes = Encoding.UTF8.GetBytes(DssePrefix); + var typeLenBytes = Encoding.UTF8.GetBytes(typeLenStr); + var payloadLenBytes = Encoding.UTF8.GetBytes(payloadLenStr); + + var totalSize = prefixBytes.Length + 1 + + typeLenBytes.Length + 1 + + typeBytes.Length + 1 + + payloadLenBytes.Length + 1 + + payload.Length; + + var pae = new byte[totalSize]; + var offset = 0; + + Buffer.BlockCopy(prefixBytes, 0, pae, offset, prefixBytes.Length); + offset += prefixBytes.Length; + pae[offset++] = 0x20; + + Buffer.BlockCopy(typeLenBytes, 0, pae, offset, typeLenBytes.Length); + offset += typeLenBytes.Length; + pae[offset++] = 0x20; + + Buffer.BlockCopy(typeBytes, 0, pae, offset, typeBytes.Length); + offset += typeBytes.Length; + pae[offset++] = 0x20; + + Buffer.BlockCopy(payloadLenBytes, 0, pae, offset, payloadLenBytes.Length); + offset += payloadLenBytes.Length; + pae[offset++] = 0x20; + + Buffer.BlockCopy(payload, 0, pae, offset, payload.Length); + + return pae; + } + + private static byte[] FromBase64Url(string value) + { + var padded = value.Replace('-', '+').Replace('_', '/'); + switch (padded.Length % 4) + { + case 2: padded += "=="; break; + case 3: padded += "="; break; + } + return Convert.FromBase64String(padded); + } + + private static async Task ComputeFileSha256Async(string path, CancellationToken ct) + { + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, useAsync: true); + var hash = await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private sealed record SignatureVerificationResult(bool IsValid, string? KeyId, ImmutableArray Errors) + { + public static SignatureVerificationResult Success(string? keyId) => new(true, keyId, []); + public static SignatureVerificationResult Failure(params string[] errors) => new(false, null, [.. errors]); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/RuleValidator.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/RuleValidator.cs new file mode 100644 index 000000000..974ac6469 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Bundles/RuleValidator.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Analyzers.Secrets.Bundles; + +/// +/// Validates secret detection rules against the schema requirements. +/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles +/// +public interface IRuleValidator +{ + /// + /// Validates a rule and returns validation errors, if any. + /// + RuleValidationResult Validate(SecretRule rule); +} + +/// +/// Result of rule validation. +/// +public sealed record RuleValidationResult +{ + /// + /// Whether the rule is valid. + /// + public required bool IsValid { get; init; } + + /// + /// Validation errors encountered. + /// + public ImmutableArray Errors { get; init; } = []; + + /// + /// Validation warnings (non-fatal). + /// + public ImmutableArray Warnings { get; init; } = []; + + public static RuleValidationResult Success() => new() { IsValid = true }; + + public static RuleValidationResult Failure(params string[] errors) => new() + { + IsValid = false, + Errors = [.. errors] + }; + + public static RuleValidationResult Failure(IEnumerable errors) => new() + { + IsValid = false, + Errors = [.. errors] + }; +} + +/// +/// Default implementation of rule validation. +/// +public sealed class RuleValidator : IRuleValidator +{ + private static readonly Regex NamespacedIdPattern = new( + @"^[a-z][a-z0-9-]*(\.[a-z][a-z0-9-]*)+$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex SemVerPattern = new( + @"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private readonly ILogger _logger; + + public RuleValidator(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public RuleValidationResult Validate(SecretRule rule) + { + ArgumentNullException.ThrowIfNull(rule); + + var errors = new List(); + var warnings = new List(); + + // Validate ID + if (string.IsNullOrWhiteSpace(rule.Id)) + { + errors.Add("Rule ID is required."); + } + else if (!NamespacedIdPattern.IsMatch(rule.Id)) + { + errors.Add($"Rule ID '{rule.Id}' must be namespaced (e.g., 'stellaops.secrets.aws-key')."); + } + + // Validate version + if (string.IsNullOrWhiteSpace(rule.Version)) + { + errors.Add("Rule version is required."); + } + else if (!SemVerPattern.IsMatch(rule.Version)) + { + errors.Add($"Rule version '{rule.Version}' must be valid SemVer."); + } + + // Validate name + if (string.IsNullOrWhiteSpace(rule.Name)) + { + warnings.Add("Rule name is recommended for documentation."); + } + + // Validate pattern for regex/composite rules + if (rule.Type is SecretRuleType.Regex or SecretRuleType.Composite) + { + if (string.IsNullOrWhiteSpace(rule.Pattern)) + { + errors.Add("Pattern is required for regex/composite rules."); + } + else + { + try + { + // Validate regex compiles + _ = new Regex(rule.Pattern, RegexOptions.Compiled, TimeSpan.FromSeconds(1)); + } + catch (RegexParseException ex) + { + errors.Add($"Invalid regex pattern: {ex.Message}"); + } + catch (ArgumentException ex) + { + errors.Add($"Invalid regex pattern: {ex.Message}"); + } + } + } + + // Validate entropy threshold for entropy rules + if (rule.Type is SecretRuleType.Entropy or SecretRuleType.Composite) + { + if (rule.EntropyThreshold <= 0 || rule.EntropyThreshold > 8) + { + warnings.Add($"Entropy threshold {rule.EntropyThreshold} may be out of typical range (3.0-6.0)."); + } + } + + // Validate min/max length + if (rule.MinLength < 0) + { + errors.Add("MinLength cannot be negative."); + } + + if (rule.MaxLength < rule.MinLength) + { + errors.Add("MaxLength must be greater than or equal to MinLength."); + } + + // Validate file patterns if provided + foreach (var pattern in rule.FilePatterns) + { + if (string.IsNullOrWhiteSpace(pattern)) + { + warnings.Add("Empty file pattern will be ignored."); + } + } + + if (errors.Count > 0) + { + _logger.LogDebug("Rule {RuleId} validation failed: {Errors}", rule.Id, string.Join("; ", errors)); + return new RuleValidationResult + { + IsValid = false, + Errors = [.. errors], + Warnings = [.. warnings] + }; + } + + if (warnings.Count > 0) + { + _logger.LogDebug("Rule {RuleId} validated with warnings: {Warnings}", rule.Id, string.Join("; ", warnings)); + } + + return new RuleValidationResult + { + IsValid = true, + Errors = [], + Warnings = [.. warnings] + }; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/CompositeSecretDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/CompositeSecretDetector.cs new file mode 100644 index 000000000..9fcb3fbd2 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/CompositeSecretDetector.cs @@ -0,0 +1,139 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Combines multiple detection strategies for comprehensive secret detection. +/// +public sealed class CompositeSecretDetector : ISecretDetector +{ + private readonly RegexDetector _regexDetector; + private readonly EntropyDetector _entropyDetector; + private readonly ILogger _logger; + + public CompositeSecretDetector( + RegexDetector regexDetector, + EntropyDetector entropyDetector, + ILogger logger) + { + _regexDetector = regexDetector ?? throw new ArgumentNullException(nameof(regexDetector)); + _entropyDetector = entropyDetector ?? throw new ArgumentNullException(nameof(entropyDetector)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string DetectorId => "composite"; + + public bool CanHandle(SecretRuleType ruleType) => true; + + public async ValueTask> DetectAsync( + ReadOnlyMemory content, + string filePath, + SecretRule rule, + CancellationToken ct = default) + { + var results = new List(); + + // Choose detector based on rule type + switch (rule.Type) + { + case SecretRuleType.Regex: + var regexMatches = await _regexDetector.DetectAsync(content, filePath, rule, ct); + results.AddRange(regexMatches); + break; + + case SecretRuleType.Entropy: + var entropyMatches = await _entropyDetector.DetectAsync(content, filePath, rule, ct); + results.AddRange(entropyMatches); + break; + + case SecretRuleType.Composite: + // Run both detectors and merge results + var regexTask = _regexDetector.DetectAsync(content, filePath, rule, ct); + var entropyTask = _entropyDetector.DetectAsync(content, filePath, rule, ct); + + var regexResults = await regexTask; + var entropyResults = await entropyTask; + + // Add regex matches + results.AddRange(regexResults); + + // Add entropy matches, boosting confidence if they overlap with regex + foreach (var entropyMatch in entropyResults) + { + var overlappingRegex = regexResults.FirstOrDefault(r => + r.LineNumber == entropyMatch.LineNumber && + OverlapsColumn(r, entropyMatch)); + + if (overlappingRegex is not null) + { + // Boost confidence for overlapping matches + results.Add(entropyMatch with + { + ConfidenceScore = Math.Min(0.99, entropyMatch.ConfidenceScore + 0.1) + }); + } + else + { + results.Add(entropyMatch); + } + } + break; + } + + // Deduplicate overlapping matches + return DeduplicateMatches(results); + } + + private static bool OverlapsColumn(SecretMatch a, SecretMatch b) + { + return a.ColumnStart <= b.ColumnEnd && b.ColumnStart <= a.ColumnEnd; + } + + private static IReadOnlyList DeduplicateMatches(List matches) + { + if (matches.Count <= 1) + { + return matches; + } + + // Sort by position + matches.Sort((a, b) => + { + var lineComp = a.LineNumber.CompareTo(b.LineNumber); + return lineComp != 0 ? lineComp : a.ColumnStart.CompareTo(b.ColumnStart); + }); + + var deduplicated = new List(); + SecretMatch? previous = null; + + foreach (var match in matches) + { + if (previous is null) + { + previous = match; + continue; + } + + // Check if this match overlaps with the previous one + if (match.LineNumber == previous.LineNumber && OverlapsColumn(previous, match)) + { + // Keep the one with higher confidence + if (match.ConfidenceScore > previous.ConfidenceScore) + { + previous = match; + } + // Otherwise keep previous + } + else + { + deduplicated.Add(previous); + previous = match; + } + } + + if (previous is not null) + { + deduplicated.Add(previous); + } + + return deduplicated; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyCalculator.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyCalculator.cs new file mode 100644 index 000000000..470f23077 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyCalculator.cs @@ -0,0 +1,161 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Calculates Shannon entropy for detecting high-entropy strings that may be secrets. +/// +public static class EntropyCalculator +{ + /// + /// Calculates Shannon entropy in bits per character for the given data. + /// + /// The data to analyze. + /// Entropy in bits per character (0.0 to 8.0 for byte data). + public static double Calculate(ReadOnlySpan data) + { + if (data.IsEmpty) + { + return 0.0; + } + + // Count occurrences of each byte value + Span counts = stackalloc int[256]; + counts.Clear(); + + foreach (byte b in data) + { + counts[b]++; + } + + // Calculate entropy using Shannon's formula + double entropy = 0.0; + double length = data.Length; + + for (int i = 0; i < 256; i++) + { + if (counts[i] > 0) + { + double probability = counts[i] / length; + entropy -= probability * Math.Log2(probability); + } + } + + return entropy; + } + + /// + /// Calculates Shannon entropy for a string. + /// + public static double Calculate(ReadOnlySpan data) + { + if (data.IsEmpty) + { + return 0.0; + } + + // For character data, we calculate based on unique characters seen + var counts = new Dictionary(); + + foreach (char c in data) + { + counts.TryGetValue(c, out int count); + counts[c] = count + 1; + } + + double entropy = 0.0; + double length = data.Length; + + foreach (var count in counts.Values) + { + double probability = count / length; + entropy -= probability * Math.Log2(probability); + } + + return entropy; + } + + /// + /// Checks if the data appears to be base64 encoded. + /// + public static bool IsBase64Like(ReadOnlySpan data) + { + if (data.Length < 4) + { + return false; + } + + int validChars = 0; + foreach (char c in data) + { + if (char.IsLetterOrDigit(c) || c is '+' or '/' or '=') + { + validChars++; + } + } + + return validChars >= data.Length * 0.9; + } + + /// + /// Checks if the data appears to be hexadecimal. + /// + public static bool IsHexLike(ReadOnlySpan data) + { + if (data.Length < 8) + { + return false; + } + + foreach (char c in data) + { + if (!char.IsAsciiHexDigit(c)) + { + return false; + } + } + + return true; + } + + /// + /// Determines if a string is likely a secret based on entropy and charset. + /// + /// The string to check. + /// Minimum entropy threshold (default 4.5). + /// True if the string appears to be a high-entropy secret. + public static bool IsLikelySecret(ReadOnlySpan data, double threshold = 4.5) + { + if (data.Length < 16) + { + return false; + } + + // Skip if it looks like a UUID (common false positive) + if (LooksLikeUuid(data)) + { + return false; + } + + var entropy = Calculate(data); + return entropy >= threshold; + } + + private static bool LooksLikeUuid(ReadOnlySpan data) + { + // UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars) + if (data.Length == 36) + { + if (data[8] == '-' && data[13] == '-' && data[18] == '-' && data[23] == '-') + { + return true; + } + } + + // UUID without dashes: 32 hex chars + if (data.Length == 32 && IsHexLike(data)) + { + return true; + } + + return false; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyDetector.cs new file mode 100644 index 000000000..323ac37fc --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/EntropyDetector.cs @@ -0,0 +1,199 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Entropy-based secret detector for high-entropy strings. +/// +public sealed class EntropyDetector : ISecretDetector +{ + private readonly ILogger _logger; + + // Regex to find potential secret strings (alphanumeric with common secret characters) + private static readonly Regex CandidatePattern = new( + @"[A-Za-z0-9+/=_\-]{16,}", + RegexOptions.Compiled | RegexOptions.CultureInvariant, + TimeSpan.FromSeconds(5)); + + public EntropyDetector(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string DetectorId => "entropy"; + + public bool CanHandle(SecretRuleType ruleType) => + ruleType is SecretRuleType.Entropy or SecretRuleType.Composite; + + public ValueTask> DetectAsync( + ReadOnlyMemory content, + string filePath, + SecretRule rule, + CancellationToken ct = default) + { + if (ct.IsCancellationRequested) + { + return ValueTask.FromResult>([]); + } + + // Decode content as UTF-8 + string text; + try + { + text = Encoding.UTF8.GetString(content.Span); + } + catch (DecoderFallbackException) + { + return ValueTask.FromResult>([]); + } + + var matches = new List(); + var lineStarts = ComputeLineStarts(text); + var threshold = rule.EntropyThreshold > 0 ? rule.EntropyThreshold : 4.5; + var minLength = rule.MinLength > 0 ? rule.MinLength : 16; + var maxLength = rule.MaxLength > 0 ? rule.MaxLength : 1000; + + try + { + foreach (Match candidate in CandidatePattern.Matches(text)) + { + if (ct.IsCancellationRequested) + { + break; + } + + var value = candidate.Value.AsSpan(); + + // Check length constraints + if (value.Length < minLength || value.Length > maxLength) + { + continue; + } + + // Skip common false positives + if (ShouldSkip(value)) + { + continue; + } + + // Calculate entropy + var entropy = EntropyCalculator.Calculate(value); + if (entropy < threshold) + { + continue; + } + + var (lineNumber, columnStart) = GetLineAndColumn(lineStarts, candidate.Index); + var matchBytes = Encoding.UTF8.GetBytes(candidate.Value); + + // Adjust confidence based on entropy level + var confidenceScore = CalculateConfidence(entropy, threshold); + + matches.Add(new SecretMatch + { + Rule = rule, + FilePath = filePath, + LineNumber = lineNumber, + ColumnStart = columnStart, + ColumnEnd = columnStart + candidate.Length - 1, + RawMatch = matchBytes, + ConfidenceScore = confidenceScore, + DetectorId = DetectorId, + Entropy = entropy + }); + } + } + catch (RegexMatchTimeoutException) + { + _logger.LogWarning( + "Entropy detection timeout on file '{FilePath}'", + filePath); + } + + return ValueTask.FromResult>(matches); + } + + private static bool ShouldSkip(ReadOnlySpan value) + { + // Skip UUIDs + if (EntropyCalculator.IsHexLike(value) && value.Length == 32) + { + return true; + } + + // Skip if it looks like a UUID with dashes + if (value.Length == 36 && value[8] == '-') + { + return true; + } + + // Skip common hash prefixes that aren't secrets + if (value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("sha512:", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("md5:", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + // Skip if it's all the same character repeated + char first = value[0]; + bool allSame = true; + for (int i = 1; i < value.Length; i++) + { + if (value[i] != first) + { + allSame = false; + break; + } + } + if (allSame) + { + return true; + } + + return false; + } + + private static double CalculateConfidence(double entropy, double threshold) + { + // Scale confidence based on how far above threshold + // entropy >= threshold + 1.5 => 0.95 (high) + // entropy >= threshold + 0.5 => 0.75 (medium) + // entropy >= threshold => 0.5 (low) + var excess = entropy - threshold; + return excess switch + { + >= 1.5 => 0.95, + >= 0.5 => 0.75, + _ => 0.5 + }; + } + + private static List ComputeLineStarts(string text) + { + var lineStarts = new List { 0 }; + for (int i = 0; i < text.Length; i++) + { + if (text[i] == '\n') + { + lineStarts.Add(i + 1); + } + } + return lineStarts; + } + + private static (int lineNumber, int column) GetLineAndColumn(List lineStarts, int position) + { + int line = 1; + for (int i = 1; i < lineStarts.Count; i++) + { + if (lineStarts[i] > position) + { + break; + } + line = i + 1; + } + + int lineStart = lineStarts[line - 1]; + int column = position - lineStart + 1; + return (line, column); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/ISecretDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/ISecretDetector.cs new file mode 100644 index 000000000..be7403242 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/ISecretDetector.cs @@ -0,0 +1,32 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Contract for secret detection strategies. +/// Implementations must be thread-safe and deterministic. +/// +public interface ISecretDetector +{ + /// + /// Unique identifier for this detector (e.g., "regex", "entropy"). + /// + string DetectorId { get; } + + /// + /// Detects secrets in the provided content using the specified rule. + /// + /// The file content to scan. + /// The file path (for reporting). + /// The rule to apply. + /// Cancellation token. + /// List of matches found. + ValueTask> DetectAsync( + ReadOnlyMemory content, + string filePath, + SecretRule rule, + CancellationToken ct = default); + + /// + /// Checks if this detector can handle the specified rule type. + /// + bool CanHandle(SecretRuleType ruleType); +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/RegexDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/RegexDetector.cs new file mode 100644 index 000000000..fcc936443 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/RegexDetector.cs @@ -0,0 +1,137 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Regex-based secret detector. +/// +public sealed class RegexDetector : ISecretDetector +{ + private readonly ILogger _logger; + + public RegexDetector(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string DetectorId => "regex"; + + public bool CanHandle(SecretRuleType ruleType) => + ruleType is SecretRuleType.Regex or SecretRuleType.Composite; + + public ValueTask> DetectAsync( + ReadOnlyMemory content, + string filePath, + SecretRule rule, + CancellationToken ct = default) + { + if (ct.IsCancellationRequested) + { + return ValueTask.FromResult>([]); + } + + var regex = rule.GetCompiledPattern(); + if (regex is null) + { + _logger.LogWarning("Rule '{RuleId}' has invalid regex pattern", rule.Id); + return ValueTask.FromResult>([]); + } + + // Decode content as UTF-8 + string text; + try + { + text = Encoding.UTF8.GetString(content.Span); + } + catch (DecoderFallbackException) + { + // Not valid UTF-8, skip + return ValueTask.FromResult>([]); + } + + // Apply keyword pre-filter + if (!rule.MightMatch(text.AsSpan())) + { + return ValueTask.FromResult>([]); + } + + var matches = new List(); + var lineStarts = ComputeLineStarts(text); + + try + { + foreach (Match match in regex.Matches(text)) + { + if (ct.IsCancellationRequested) + { + break; + } + + if (match.Length < rule.MinLength || match.Length > rule.MaxLength) + { + continue; + } + + var (lineNumber, columnStart) = GetLineAndColumn(lineStarts, match.Index); + var matchBytes = Encoding.UTF8.GetBytes(match.Value); + + matches.Add(new SecretMatch + { + Rule = rule, + FilePath = filePath, + LineNumber = lineNumber, + ColumnStart = columnStart, + ColumnEnd = columnStart + match.Length - 1, + RawMatch = matchBytes, + ConfidenceScore = MapConfidenceToScore(rule.Confidence), + DetectorId = DetectorId + }); + } + } + catch (RegexMatchTimeoutException) + { + _logger.LogWarning( + "Regex timeout for rule '{RuleId}' on file '{FilePath}'", + rule.Id, + filePath); + } + + return ValueTask.FromResult>(matches); + } + + private static List ComputeLineStarts(string text) + { + var lineStarts = new List { 0 }; + for (int i = 0; i < text.Length; i++) + { + if (text[i] == '\n') + { + lineStarts.Add(i + 1); + } + } + return lineStarts; + } + + private static (int lineNumber, int column) GetLineAndColumn(List lineStarts, int position) + { + int line = 1; + for (int i = 1; i < lineStarts.Count; i++) + { + if (lineStarts[i] > position) + { + break; + } + line = i + 1; + } + + int lineStart = lineStarts[line - 1]; + int column = position - lineStart + 1; + return (line, column); + } + + private static double MapConfidenceToScore(SecretConfidence confidence) => confidence switch + { + SecretConfidence.Low => 0.5, + SecretConfidence.Medium => 0.75, + SecretConfidence.High => 0.95, + _ => 0.5 + }; +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/SecretMatch.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/SecretMatch.cs new file mode 100644 index 000000000..d0c079a53 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Detectors/SecretMatch.cs @@ -0,0 +1,57 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Represents a potential secret match found by a detector. +/// +public sealed record SecretMatch +{ + /// + /// The rule that produced this match. + /// + public required SecretRule Rule { get; init; } + + /// + /// The file path where the match was found. + /// + public required string FilePath { get; init; } + + /// + /// The 1-based line number of the match. + /// + public required int LineNumber { get; init; } + + /// + /// The 1-based column where the match starts. + /// + public required int ColumnStart { get; init; } + + /// + /// The 1-based column where the match ends. + /// + public required int ColumnEnd { get; init; } + + /// + /// The raw matched content (will be masked before output). + /// + public required ReadOnlyMemory RawMatch { get; init; } + + /// + /// Confidence score from 0.0 to 1.0. + /// + public required double ConfidenceScore { get; init; } + + /// + /// The detector that found this match. + /// + public required string DetectorId { get; init; } + + /// + /// Optional entropy value if entropy-based detection was used. + /// + public double? Entropy { get; init; } + + /// + /// Gets the length of the matched content. + /// + public int MatchLength => RawMatch.Length; +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretFinding.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretFinding.cs new file mode 100644 index 000000000..5b36d918a --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretFinding.cs @@ -0,0 +1,52 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Aggregated finding for storage in ScanAnalysisStore. +/// +public sealed record SecretFinding +{ + /// + /// Unique identifier for this finding. + /// + public required Guid Id { get; init; } + + /// + /// The evidence record. + /// + public required SecretLeakEvidence Evidence { get; init; } + + /// + /// The scan that produced this finding. + /// + public required string ScanId { get; init; } + + /// + /// The tenant that owns this finding. + /// + public required string TenantId { get; init; } + + /// + /// The artifact digest (container image or other artifact). + /// + public required string ArtifactDigest { get; init; } + + /// + /// Creates a new finding from evidence. + /// + public static SecretFinding Create( + SecretLeakEvidence evidence, + string scanId, + string tenantId, + string artifactDigest, + Guid? id = null) + { + return new SecretFinding + { + Id = id ?? Guid.NewGuid(), + Evidence = evidence, + ScanId = scanId, + TenantId = tenantId, + ArtifactDigest = artifactDigest + }; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretLeakEvidence.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretLeakEvidence.cs new file mode 100644 index 000000000..874164709 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Evidence/SecretLeakEvidence.cs @@ -0,0 +1,136 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Evidence record for a detected secret leak. +/// This record is emitted to the policy engine for decision-making. +/// +public sealed record SecretLeakEvidence +{ + /// + /// The evidence type identifier. + /// + public const string EvidenceType = "secret.leak"; + + /// + /// The rule ID that produced this finding. + /// + public required string RuleId { get; init; } + + /// + /// The rule version. + /// + public required string RuleVersion { get; init; } + + /// + /// The severity of the finding. + /// + public required SecretSeverity Severity { get; init; } + + /// + /// The confidence level of the finding. + /// + public required SecretConfidence Confidence { get; init; } + + /// + /// The file path where the secret was found (relative to scan root). + /// + public required string FilePath { get; init; } + + /// + /// The 1-based line number. + /// + public required int LineNumber { get; init; } + + /// + /// The 1-based column number. + /// + public int ColumnNumber { get; init; } + + /// + /// The masked payload (e.g., "AKIA****B7"). Never contains the actual secret. + /// + public required string Mask { get; init; } + + /// + /// The bundle ID that contained the rule. + /// + public required string BundleId { get; init; } + + /// + /// The bundle version. + /// + public required string BundleVersion { get; init; } + + /// + /// When this finding was detected. + /// + public required DateTimeOffset DetectedAt { get; init; } + + /// + /// The detector that found this secret. + /// + public required string DetectorId { get; init; } + + /// + /// Entropy value if entropy-based detection was used. + /// + public double? Entropy { get; init; } + + /// + /// Additional metadata for the finding. + /// + public ImmutableDictionary Metadata { get; init; } = + ImmutableDictionary.Empty; + + /// + /// Creates evidence from a secret match and masker. + /// + public static SecretLeakEvidence FromMatch( + SecretMatch match, + IPayloadMasker masker, + SecretRuleset ruleset, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(match); + ArgumentNullException.ThrowIfNull(masker); + ArgumentNullException.ThrowIfNull(ruleset); + ArgumentNullException.ThrowIfNull(timeProvider); + + var masked = masker.Mask(match.RawMatch.Span, match.Rule.MaskingHint); + + return new SecretLeakEvidence + { + RuleId = match.Rule.Id, + RuleVersion = match.Rule.Version, + Severity = match.Rule.Severity, + Confidence = MapScoreToConfidence(match.ConfidenceScore, match.Rule.Confidence), + FilePath = match.FilePath, + LineNumber = match.LineNumber, + ColumnNumber = match.ColumnStart, + Mask = masked, + BundleId = ruleset.Id, + BundleVersion = ruleset.Version, + DetectedAt = timeProvider.GetUtcNow(), + DetectorId = match.DetectorId, + Entropy = match.Entropy + }; + } + + private static SecretConfidence MapScoreToConfidence(double score, SecretConfidence ruleDefault) + { + // Adjust confidence based on detection score + if (score >= 0.9) + { + return SecretConfidence.High; + } + if (score >= 0.7) + { + return SecretConfidence.Medium; + } + if (score >= 0.5) + { + return ruleDefault; + } + return SecretConfidence.Low; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/GlobalUsings.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/GlobalUsings.cs new file mode 100644 index 000000000..3e45ebedc --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/GlobalUsings.cs @@ -0,0 +1,10 @@ +global using System.Collections.Immutable; +global using System.Diagnostics.CodeAnalysis; +global using System.Globalization; +global using System.Text; +global using System.Text.Json; +global using System.Text.RegularExpressions; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.DependencyInjection.Extensions; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/IPayloadMasker.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/IPayloadMasker.cs new file mode 100644 index 000000000..e340a302a --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/IPayloadMasker.cs @@ -0,0 +1,23 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Contract for secret payload masking. +/// +public interface IPayloadMasker +{ + /// + /// Masks a secret payload, preserving prefix/suffix for identification. + /// + /// The raw secret bytes. + /// Optional masking hint (e.g., "prefix:4,suffix:2"). + /// Masked string (e.g., "AKIA****B7"). + string Mask(ReadOnlySpan payload, string? hint = null); + + /// + /// Masks a secret string, preserving prefix/suffix for identification. + /// + /// The raw secret string. + /// Optional masking hint (e.g., "prefix:4,suffix:2"). + /// Masked string (e.g., "AKIA****B7"). + string Mask(ReadOnlySpan payload, string? hint = null); +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/PayloadMasker.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/PayloadMasker.cs new file mode 100644 index 000000000..40e939702 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Masking/PayloadMasker.cs @@ -0,0 +1,151 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Default implementation of payload masking for secrets. +/// +public sealed class PayloadMasker : IPayloadMasker +{ + /// + /// Default number of characters to preserve at the start. + /// + public const int DefaultPrefixLength = 4; + + /// + /// Default number of characters to preserve at the end. + /// + public const int DefaultSuffixLength = 2; + + /// + /// Maximum number of mask characters to use. + /// + public const int MaxMaskLength = 8; + + /// + /// Minimum output length for masked values. + /// + public const int MinOutputLength = 8; + + /// + /// Maximum total characters to expose (prefix + suffix). + /// + public const int MaxExposedChars = 6; + + /// + /// The character used for masking. + /// + public const char MaskChar = '*'; + + public string Mask(ReadOnlySpan payload, string? hint = null) + { + if (payload.IsEmpty) + { + return string.Empty; + } + + // Try to decode as UTF-8 + try + { + var text = Encoding.UTF8.GetString(payload); + return Mask(text.AsSpan(), hint); + } + catch (DecoderFallbackException) + { + // Not valid UTF-8, represent as hex + var hex = Convert.ToHexString(payload); + return Mask(hex.AsSpan(), hint); + } + } + + public string Mask(ReadOnlySpan payload, string? hint = null) + { + if (payload.IsEmpty) + { + return string.Empty; + } + + var (prefixLen, suffixLen) = ParseHint(hint); + + // Enforce maximum exposed characters + if (prefixLen + suffixLen > MaxExposedChars) + { + var ratio = (double)prefixLen / (prefixLen + suffixLen); + prefixLen = (int)(MaxExposedChars * ratio); + suffixLen = MaxExposedChars - prefixLen; + } + + // Handle short payloads + if (payload.Length <= prefixLen + suffixLen) + { + // Too short to mask meaningfully, just return masked placeholder + return new string(MaskChar, Math.Min(payload.Length, MinOutputLength)); + } + + // Calculate mask length + var middleLength = payload.Length - prefixLen - suffixLen; + var maskLength = Math.Min(middleLength, MaxMaskLength); + + // Build masked output + var sb = new StringBuilder(prefixLen + maskLength + suffixLen); + + // Prefix + if (prefixLen > 0) + { + sb.Append(payload[..prefixLen]); + } + + // Mask + sb.Append(MaskChar, maskLength); + + // Suffix + if (suffixLen > 0) + { + sb.Append(payload[^suffixLen..]); + } + + // Ensure minimum length + while (sb.Length < MinOutputLength) + { + sb.Insert(prefixLen, MaskChar); + } + + return sb.ToString(); + } + + private static (int prefix, int suffix) ParseHint(string? hint) + { + if (string.IsNullOrWhiteSpace(hint)) + { + return (DefaultPrefixLength, DefaultSuffixLength); + } + + int prefix = DefaultPrefixLength; + int suffix = DefaultSuffixLength; + + // Parse hint format: "prefix:4,suffix:2" + var parts = hint.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var part in parts) + { + var kv = part.Split(':', StringSplitOptions.TrimEntries); + if (kv.Length != 2) + { + continue; + } + + if (!int.TryParse(kv[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var value)) + { + continue; + } + + if (kv[0].Equals("prefix", StringComparison.OrdinalIgnoreCase)) + { + prefix = Math.Max(0, Math.Min(value, MaxExposedChars)); + } + else if (kv[0].Equals("suffix", StringComparison.OrdinalIgnoreCase)) + { + suffix = Math.Max(0, Math.Min(value, MaxExposedChars)); + } + } + + return (prefix, suffix); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/IRulesetLoader.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/IRulesetLoader.cs new file mode 100644 index 000000000..03cd9b354 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/IRulesetLoader.cs @@ -0,0 +1,29 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Contract for loading secret detection rulesets. +/// +public interface IRulesetLoader +{ + /// + /// Loads a ruleset from a directory containing bundle files. + /// + /// Path to the bundle directory. + /// Cancellation token. + /// The loaded ruleset. + ValueTask LoadAsync(string bundlePath, CancellationToken ct = default); + + /// + /// Loads a ruleset from a JSONL stream. + /// + /// Stream containing NDJSON rule definitions. + /// The bundle identifier. + /// The bundle version. + /// Cancellation token. + /// The loaded ruleset. + ValueTask LoadFromJsonlAsync( + Stream rulesStream, + string bundleId, + string bundleVersion, + CancellationToken ct = default); +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/RulesetLoader.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/RulesetLoader.cs new file mode 100644 index 000000000..d9cba705f --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/RulesetLoader.cs @@ -0,0 +1,227 @@ +using System.Security.Cryptography; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Loads secret detection rulesets from bundle files. +/// +public sealed class RulesetLoader : IRulesetLoader +{ + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + Converters = + { + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) + } + }; + + public RulesetLoader(ILogger logger, TimeProvider timeProvider) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public async ValueTask LoadAsync(string bundlePath, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(bundlePath)) + { + throw new ArgumentException("Bundle path is required", nameof(bundlePath)); + } + + if (!Directory.Exists(bundlePath)) + { + throw new DirectoryNotFoundException($"Bundle directory not found: {bundlePath}"); + } + + // Load manifest + var manifestPath = Path.Combine(bundlePath, "secrets.ruleset.manifest.json"); + if (!File.Exists(manifestPath)) + { + throw new FileNotFoundException("Bundle manifest not found", manifestPath); + } + + var manifestJson = await File.ReadAllTextAsync(manifestPath, ct); + var manifest = JsonSerializer.Deserialize(manifestJson, JsonOptions) + ?? throw new InvalidOperationException("Failed to parse bundle manifest"); + + // Load rules + var rulesPath = Path.Combine(bundlePath, "secrets.ruleset.rules.jsonl"); + if (!File.Exists(rulesPath)) + { + throw new FileNotFoundException("Bundle rules file not found", rulesPath); + } + + await using var rulesStream = File.OpenRead(rulesPath); + var ruleset = await LoadFromJsonlAsync( + rulesStream, + manifest.Id ?? "secrets.ruleset", + manifest.Version ?? "unknown", + ct); + + // Verify integrity if digest is available + if (!string.IsNullOrEmpty(manifest.Integrity?.RulesSha256)) + { + rulesStream.Position = 0; + var actualDigest = await ComputeSha256Async(rulesStream, ct); + if (!string.Equals(actualDigest, manifest.Integrity.RulesSha256, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"Rules file integrity check failed. Expected: {manifest.Integrity.RulesSha256}, Actual: {actualDigest}"); + } + } + + _logger.LogInformation( + "Loaded secrets ruleset '{BundleId}' version {Version} with {RuleCount} rules ({EnabledCount} enabled)", + ruleset.Id, + ruleset.Version, + ruleset.Rules.Length, + ruleset.EnabledRuleCount); + + return ruleset; + } + + public async ValueTask LoadFromJsonlAsync( + Stream rulesStream, + string bundleId, + string bundleVersion, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(rulesStream); + + var rules = new List(); + using var reader = new StreamReader(rulesStream, Encoding.UTF8, leaveOpen: true); + + int lineNumber = 0; + string? line; + while ((line = await reader.ReadLineAsync(ct)) is not null) + { + ct.ThrowIfCancellationRequested(); + lineNumber++; + + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + try + { + var ruleJson = JsonSerializer.Deserialize(line, JsonOptions); + if (ruleJson is null) + { + _logger.LogWarning("Skipping null rule at line {LineNumber}", lineNumber); + continue; + } + + var rule = MapToRule(ruleJson); + rules.Add(rule); + } + catch (JsonException ex) + { + _logger.LogWarning(ex, "Failed to parse rule at line {LineNumber}", lineNumber); + } + } + + // Sort rules by ID for deterministic ordering + rules.Sort((a, b) => string.Compare(a.Id, b.Id, StringComparison.Ordinal)); + + return new SecretRuleset + { + Id = bundleId, + Version = bundleVersion, + CreatedAt = _timeProvider.GetUtcNow(), + Rules = [.. rules] + }; + } + + private static SecretRule MapToRule(RuleJson json) + { + return new SecretRule + { + Id = json.Id ?? throw new InvalidOperationException("Rule ID is required"), + Version = json.Version ?? "1.0.0", + Name = json.Name ?? json.Id ?? "Unknown", + Description = json.Description ?? string.Empty, + Type = ParseRuleType(json.Type), + Pattern = json.Pattern ?? throw new InvalidOperationException("Rule pattern is required"), + Severity = ParseSeverity(json.Severity), + Confidence = ParseConfidence(json.Confidence), + MaskingHint = json.MaskingHint, + Keywords = json.Keywords?.ToImmutableArray() ?? [], + FilePatterns = json.FilePatterns?.ToImmutableArray() ?? [], + Enabled = json.Enabled ?? true, + EntropyThreshold = json.EntropyThreshold ?? 4.5, + MinLength = json.MinLength ?? 16, + MaxLength = json.MaxLength ?? 1000, + Metadata = json.Metadata?.ToImmutableDictionary() ?? ImmutableDictionary.Empty + }; + } + + private static SecretRuleType ParseRuleType(string? type) => type?.ToLowerInvariant() switch + { + "regex" => SecretRuleType.Regex, + "entropy" => SecretRuleType.Entropy, + "composite" => SecretRuleType.Composite, + _ => SecretRuleType.Regex + }; + + private static SecretSeverity ParseSeverity(string? severity) => severity?.ToLowerInvariant() switch + { + "low" => SecretSeverity.Low, + "medium" => SecretSeverity.Medium, + "high" => SecretSeverity.High, + "critical" => SecretSeverity.Critical, + _ => SecretSeverity.Medium + }; + + private static SecretConfidence ParseConfidence(string? confidence) => confidence?.ToLowerInvariant() switch + { + "low" => SecretConfidence.Low, + "medium" => SecretConfidence.Medium, + "high" => SecretConfidence.High, + _ => SecretConfidence.Medium + }; + + private static async Task ComputeSha256Async(Stream stream, CancellationToken ct) + { + var hash = await SHA256.HashDataAsync(stream, ct); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + // JSON deserialization models + private sealed class BundleManifest + { + public string? Id { get; set; } + public string? Version { get; set; } + public IntegrityInfo? Integrity { get; set; } + } + + private sealed class IntegrityInfo + { + public string? RulesSha256 { get; set; } + } + + private sealed class RuleJson + { + public string? Id { get; set; } + public string? Version { get; set; } + public string? Name { get; set; } + public string? Description { get; set; } + public string? Type { get; set; } + public string? Pattern { get; set; } + public string? Severity { get; set; } + public string? Confidence { get; set; } + public string? MaskingHint { get; set; } + public List? Keywords { get; set; } + public List? FilePatterns { get; set; } + public bool? Enabled { get; set; } + public double? EntropyThreshold { get; set; } + public int? MinLength { get; set; } + public int? MaxLength { get; set; } + public Dictionary? Metadata { get; set; } + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretConfidence.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretConfidence.cs new file mode 100644 index 000000000..c87cd2971 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretConfidence.cs @@ -0,0 +1,22 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Confidence level for a secret detection finding. +/// +public enum SecretConfidence +{ + /// + /// Low confidence - may be a false positive. + /// + Low = 0, + + /// + /// Medium confidence - likely a real secret but requires verification. + /// + Medium = 1, + + /// + /// High confidence - almost certainly a real secret. + /// + High = 2 +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRule.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRule.cs new file mode 100644 index 000000000..01c2462af --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRule.cs @@ -0,0 +1,191 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// A single secret detection rule defining patterns and metadata for identifying secrets. +/// +public sealed record SecretRule +{ + /// + /// Unique rule identifier (e.g., "stellaops.secrets.aws-access-key"). + /// + public required string Id { get; init; } + + /// + /// Rule version in SemVer format (e.g., "1.0.0"). + /// + public required string Version { get; init; } + + /// + /// Human-readable rule name. + /// + public required string Name { get; init; } + + /// + /// Detailed description of what this rule detects. + /// + public required string Description { get; init; } + + /// + /// The detection strategy type. + /// + public required SecretRuleType Type { get; init; } + + /// + /// The detection pattern (regex pattern for Regex type, entropy config for Entropy type). + /// + public required string Pattern { get; init; } + + /// + /// Default severity for findings from this rule. + /// + public required SecretSeverity Severity { get; init; } + + /// + /// Default confidence level for findings from this rule. + /// + public required SecretConfidence Confidence { get; init; } + + /// + /// Optional masking hint (e.g., "prefix:4,suffix:2") for payload masking. + /// + public string? MaskingHint { get; init; } + + /// + /// Pre-filter keywords for fast rejection of non-matching content. + /// + public ImmutableArray Keywords { get; init; } = []; + + /// + /// Glob patterns for files this rule should be applied to. + /// Empty means all text files. + /// + public ImmutableArray FilePatterns { get; init; } = []; + + /// + /// Whether this rule is enabled. + /// + public bool Enabled { get; init; } = true; + + /// + /// Minimum entropy threshold for entropy-based detection. + /// Only used when Type is Entropy or Composite. + /// + public double EntropyThreshold { get; init; } = 4.5; + + /// + /// Minimum string length for entropy-based detection. + /// + public int MinLength { get; init; } = 16; + + /// + /// Maximum string length for detection (prevents matching entire files). + /// + public int MaxLength { get; init; } = 1000; + + /// + /// Optional metadata for the rule. + /// + public ImmutableDictionary Metadata { get; init; } = + ImmutableDictionary.Empty; + + /// + /// The compiled regex pattern, created lazily. + /// + private Regex? _compiledPattern; + + /// + /// Gets the compiled regex for this rule. Returns null if the pattern is invalid. + /// + public Regex? GetCompiledPattern() + { + if (Type == SecretRuleType.Entropy) + { + return null; + } + + if (_compiledPattern is not null) + { + return _compiledPattern; + } + + try + { + _compiledPattern = new Regex( + Pattern, + RegexOptions.Compiled | RegexOptions.CultureInvariant, + TimeSpan.FromSeconds(5)); + return _compiledPattern; + } + catch (ArgumentException) + { + return null; + } + } + + /// + /// Checks if the content might match this rule based on keywords. + /// Returns true if no keywords are defined or if any keyword is found. + /// + public bool MightMatch(ReadOnlySpan content) + { + if (Keywords.IsDefaultOrEmpty) + { + return true; + } + + foreach (var keyword in Keywords) + { + if (content.Contains(keyword.AsSpan(), StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + /// + /// Checks if this rule should be applied to the given file path. + /// + public bool AppliesToFile(string filePath) + { + if (FilePatterns.IsDefaultOrEmpty) + { + return true; + } + + var fileName = Path.GetFileName(filePath); + foreach (var pattern in FilePatterns) + { + if (MatchesGlob(fileName, pattern) || MatchesGlob(filePath, pattern)) + { + return true; + } + } + + return false; + } + + private static bool MatchesGlob(string path, string pattern) + { + // Simple glob matching for common patterns + if (pattern.StartsWith("**", StringComparison.Ordinal)) + { + var suffix = pattern[2..].TrimStart('/').TrimStart('\\'); + if (suffix.StartsWith("*.", StringComparison.Ordinal)) + { + var extension = suffix[1..]; + return path.EndsWith(extension, StringComparison.OrdinalIgnoreCase); + } + return path.Contains(suffix, StringComparison.OrdinalIgnoreCase); + } + + if (pattern.StartsWith("*.", StringComparison.Ordinal)) + { + var extension = pattern[1..]; + return path.EndsWith(extension, StringComparison.OrdinalIgnoreCase); + } + + return path.Equals(pattern, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleType.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleType.cs new file mode 100644 index 000000000..92b03f027 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleType.cs @@ -0,0 +1,22 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// The type of detection strategy used by a secret rule. +/// +public enum SecretRuleType +{ + /// + /// Regex-based pattern matching. + /// + Regex = 0, + + /// + /// Shannon entropy-based detection for high-entropy strings. + /// + Entropy = 1, + + /// + /// Combined regex and entropy detection. + /// + Composite = 2 +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleset.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleset.cs new file mode 100644 index 000000000..2a373fdba --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretRuleset.cs @@ -0,0 +1,115 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// A versioned collection of secret detection rules. +/// +public sealed record SecretRuleset +{ + /// + /// Bundle identifier (e.g., "secrets.ruleset"). + /// + public required string Id { get; init; } + + /// + /// Bundle version in YYYY.MM format (e.g., "2026.01"). + /// + public required string Version { get; init; } + + /// + /// When this bundle was created. + /// + public required DateTimeOffset CreatedAt { get; init; } + + /// + /// The rules in this bundle. + /// + public required ImmutableArray Rules { get; init; } + + /// + /// SHA-256 digest of the rules file for integrity verification. + /// + public string? Sha256Digest { get; init; } + + /// + /// Optional description of this bundle. + /// + public string? Description { get; init; } + + /// + /// Gets only the enabled rules from this bundle. + /// + public IEnumerable EnabledRules => Rules.Where(r => r.Enabled); + + /// + /// Gets the count of enabled rules. + /// + public int EnabledRuleCount => Rules.Count(r => r.Enabled); + + /// + /// Creates an empty ruleset. + /// + public static SecretRuleset Empty { get; } = new() + { + Id = "empty", + Version = "0.0", + CreatedAt = DateTimeOffset.MinValue, + Rules = [] + }; + + /// + /// Validates that all rules in this bundle have valid patterns. + /// + /// A list of validation errors, empty if valid. + public IReadOnlyList Validate() + { + var errors = new List(); + + if (string.IsNullOrWhiteSpace(Id)) + { + errors.Add("Bundle ID is required"); + } + + if (string.IsNullOrWhiteSpace(Version)) + { + errors.Add("Bundle version is required"); + } + + var seenIds = new HashSet(StringComparer.Ordinal); + foreach (var rule in Rules) + { + if (string.IsNullOrWhiteSpace(rule.Id)) + { + errors.Add("Rule ID is required"); + continue; + } + + if (!seenIds.Add(rule.Id)) + { + errors.Add($"Duplicate rule ID: {rule.Id}"); + } + + if (string.IsNullOrWhiteSpace(rule.Pattern)) + { + errors.Add($"Rule '{rule.Id}' has no pattern"); + } + + if (rule.Type is SecretRuleType.Regex or SecretRuleType.Composite) + { + if (rule.GetCompiledPattern() is null) + { + errors.Add($"Rule '{rule.Id}' has invalid regex pattern"); + } + } + } + + return errors; + } + + /// + /// Gets rules that apply to the specified file. + /// + public IEnumerable GetRulesForFile(string filePath) + { + return EnabledRules.Where(r => r.AppliesToFile(filePath)); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretSeverity.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretSeverity.cs new file mode 100644 index 000000000..a8563d5dc --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/Rules/SecretSeverity.cs @@ -0,0 +1,27 @@ +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Severity level for a secret detection rule. +/// +public enum SecretSeverity +{ + /// + /// Low severity - informational or low-risk credentials. + /// + Low = 0, + + /// + /// Medium severity - credentials with limited scope or short lifespan. + /// + Medium = 1, + + /// + /// High severity - production credentials with broad access. + /// + High = 2, + + /// + /// Critical severity - highly privileged credentials requiring immediate action. + /// + Critical = 3 +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs new file mode 100644 index 000000000..cfbb94986 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzer.cs @@ -0,0 +1,234 @@ +using StellaOps.Scanner.Analyzers.Lang; + +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Analyzer that detects accidentally committed secrets in container layers. +/// +public sealed class SecretsAnalyzer : ILanguageAnalyzer +{ + private readonly IOptions _options; + private readonly CompositeSecretDetector _detector; + private readonly IPayloadMasker _masker; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + private SecretRuleset? _ruleset; + + public SecretsAnalyzer( + IOptions options, + CompositeSecretDetector detector, + IPayloadMasker masker, + ILogger logger, + TimeProvider timeProvider) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _detector = detector ?? throw new ArgumentNullException(nameof(detector)); + _masker = masker ?? throw new ArgumentNullException(nameof(masker)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public string Id => "secrets"; + public string DisplayName => "Secret Leak Detector"; + + /// + /// Gets whether the analyzer is enabled and has a valid ruleset. + /// + public bool IsEnabled => _options.Value.Enabled && _ruleset is not null; + + /// + /// Gets the currently loaded ruleset. + /// + public SecretRuleset? Ruleset => _ruleset; + + /// + /// Sets the ruleset to use for detection. + /// Called by SecretsAnalyzerHost after loading the bundle. + /// + internal void SetRuleset(SecretRuleset ruleset) + { + _ruleset = ruleset ?? throw new ArgumentNullException(nameof(ruleset)); + } + + public async ValueTask AnalyzeAsync( + LanguageAnalyzerContext context, + LanguageComponentWriter writer, + CancellationToken cancellationToken) + { + if (!IsEnabled) + { + _logger.LogDebug("Secrets analyzer is disabled or has no ruleset"); + return; + } + + var options = _options.Value; + var findings = new List(); + var filesScanned = 0; + + // Scan all text files in the root + foreach (var filePath in EnumerateTextFiles(context.RootPath, options)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (findings.Count >= options.MaxFindingsPerScan) + { + _logger.LogWarning( + "Maximum findings limit ({MaxFindings}) reached, stopping scan", + options.MaxFindingsPerScan); + break; + } + + var fileFindings = await ScanFileAsync(context, filePath, options, cancellationToken); + findings.AddRange(fileFindings); + filesScanned++; + } + + _logger.LogInformation( + "Secrets scan complete: {FileCount} files scanned, {FindingCount} findings", + filesScanned, + findings.Count); + + // Store findings in analysis store if available + if (context.AnalysisStore is not null && findings.Count > 0) + { + await StoreFindings(context.AnalysisStore, findings, cancellationToken); + } + } + + private async ValueTask> ScanFileAsync( + LanguageAnalyzerContext context, + string filePath, + SecretsAnalyzerOptions options, + CancellationToken ct) + { + var findings = new List(); + + try + { + var fileInfo = new FileInfo(filePath); + if (fileInfo.Length > options.MaxFileSizeBytes) + { + _logger.LogDebug("Skipping large file: {FilePath} ({Size} bytes)", filePath, fileInfo.Length); + return findings; + } + + var content = await File.ReadAllBytesAsync(filePath, ct); + var relativePath = context.GetRelativePath(filePath); + + foreach (var rule in _ruleset!.GetRulesForFile(relativePath)) + { + ct.ThrowIfCancellationRequested(); + + var matches = await _detector.DetectAsync(content, relativePath, rule, ct); + + foreach (var match in matches) + { + // Check confidence threshold + var confidence = MapScoreToConfidence(match.ConfidenceScore); + if (confidence < options.MinConfidence) + { + continue; + } + + var evidence = SecretLeakEvidence.FromMatch(match, _masker, _ruleset, _timeProvider); + findings.Add(evidence); + + _logger.LogDebug( + "Found secret: Rule={RuleId}, File={FilePath}:{Line}, Mask={Mask}", + rule.Id, + relativePath, + match.LineNumber, + evidence.Mask); + } + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Error scanning file: {FilePath}", filePath); + } + + return findings; + } + + private static IEnumerable EnumerateTextFiles(string rootPath, SecretsAnalyzerOptions options) + { + var searchOptions = new EnumerationOptions + { + RecurseSubdirectories = true, + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.System | FileAttributes.Hidden + }; + + foreach (var file in Directory.EnumerateFiles(rootPath, "*", searchOptions)) + { + var extension = Path.GetExtension(file).ToLowerInvariant(); + + // Check exclusions + if (options.ExcludeExtensions.Contains(extension)) + { + continue; + } + + // Check if directory is excluded + var relativePath = Path.GetRelativePath(rootPath, file).Replace('\\', '/'); + if (IsExcludedDirectory(relativePath, options.ExcludeDirectories)) + { + continue; + } + + // Check inclusions if specified + if (options.IncludeExtensions.Count > 0 && !options.IncludeExtensions.Contains(extension)) + { + continue; + } + + yield return file; + } + } + + private static bool IsExcludedDirectory(string relativePath, HashSet patterns) + { + foreach (var pattern in patterns) + { + if (MatchesGlobPattern(relativePath, pattern)) + { + return true; + } + } + return false; + } + + private static bool MatchesGlobPattern(string path, string pattern) + { + if (pattern.StartsWith("**/", StringComparison.Ordinal)) + { + var suffix = pattern[3..]; + if (suffix.EndsWith("/**", StringComparison.Ordinal)) + { + var middle = suffix[..^3]; + return path.Contains(middle, StringComparison.OrdinalIgnoreCase); + } + return path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase); + } + return path.StartsWith(pattern, StringComparison.OrdinalIgnoreCase); + } + + private static SecretConfidence MapScoreToConfidence(double score) => score switch + { + >= 0.9 => SecretConfidence.High, + >= 0.7 => SecretConfidence.Medium, + _ => SecretConfidence.Low + }; + + private async ValueTask StoreFindings( + object analysisStore, + List findings, + CancellationToken ct) + { + // TODO: Store findings in ScanAnalysisStore when interface is defined + // For now, just log that we would store them + _logger.LogDebug("Would store {Count} secret findings in analysis store", findings.Count); + await ValueTask.CompletedTask; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerHost.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerHost.cs new file mode 100644 index 000000000..2ab69dfdd --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerHost.cs @@ -0,0 +1,186 @@ +using Microsoft.Extensions.Hosting; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; + +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Hosted service that manages the lifecycle of the secrets analyzer. +/// Loads and validates the rule bundle on startup with optional signature verification. +/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles +/// +public sealed class SecretsAnalyzerHost : IHostedService +{ + private readonly SecretsAnalyzer _analyzer; + private readonly IRulesetLoader _rulesetLoader; + private readonly IBundleVerifier? _bundleVerifier; + private readonly IOptions _options; + private readonly ILogger _logger; + + public SecretsAnalyzerHost( + SecretsAnalyzer analyzer, + IRulesetLoader rulesetLoader, + IOptions options, + ILogger logger, + IBundleVerifier? bundleVerifier = null) + { + _analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer)); + _rulesetLoader = rulesetLoader ?? throw new ArgumentNullException(nameof(rulesetLoader)); + _bundleVerifier = bundleVerifier; + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Gets the bundle verification result from the last startup, if available. + /// + public BundleVerificationResult? LastVerificationResult { get; private set; } + + /// + /// Gets whether the analyzer is enabled and has loaded successfully. + /// + public bool IsEnabled => _analyzer.IsEnabled; + + /// + /// Gets the loaded bundle version, if available. + /// + public string? BundleVersion => _analyzer.Ruleset?.Version; + + public async Task StartAsync(CancellationToken cancellationToken) + { + var options = _options.Value; + + if (!options.Enabled) + { + _logger.LogInformation("SecretsAnalyzerHost: Secret leak detection is disabled"); + return; + } + + _logger.LogInformation("SecretsAnalyzerHost: Loading secrets rule bundle from {Path}", options.RulesetPath); + + try + { + // Verify bundle signature if required + if (options.RequireSignatureVerification || _bundleVerifier is not null) + { + await VerifyBundleAsync(options, cancellationToken); + } + + var ruleset = await _rulesetLoader.LoadAsync(options.RulesetPath, cancellationToken); + + // Validate the ruleset + var errors = ruleset.Validate(); + if (errors.Count > 0) + { + _logger.LogError( + "SecretsAnalyzerHost: Bundle validation failed with {ErrorCount} errors: {Errors}", + errors.Count, + string.Join(", ", errors)); + + if (options.FailOnInvalidBundle) + { + throw new InvalidOperationException( + $"Secret detection bundle validation failed: {string.Join(", ", errors)}"); + } + + return; + } + + // Set the ruleset on the analyzer + _analyzer.SetRuleset(ruleset); + + _logger.LogInformation( + "SecretsAnalyzerHost: Loaded bundle '{BundleId}' version {Version} with {RuleCount} rules ({EnabledCount} enabled)", + ruleset.Id, + ruleset.Version, + ruleset.Rules.Length, + ruleset.EnabledRuleCount); + } + catch (DirectoryNotFoundException ex) + { + _logger.LogWarning(ex, "SecretsAnalyzerHost: Bundle directory not found, analyzer disabled"); + + if (options.FailOnInvalidBundle) + { + throw; + } + } + catch (FileNotFoundException ex) + { + _logger.LogWarning(ex, "SecretsAnalyzerHost: Bundle file not found, analyzer disabled"); + + if (options.FailOnInvalidBundle) + { + throw; + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogError(ex, "SecretsAnalyzerHost: Failed to load bundle"); + + if (options.FailOnInvalidBundle) + { + throw; + } + } + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("SecretsAnalyzerHost: Shutting down"); + return Task.CompletedTask; + } + + private async Task VerifyBundleAsync(SecretsAnalyzerOptions options, CancellationToken ct) + { + if (_bundleVerifier is null) + { + if (options.RequireSignatureVerification) + { + throw new InvalidOperationException( + "Signature verification is required but no IBundleVerifier is registered."); + } + return; + } + + var verificationOptions = new BundleVerificationOptions + { + RequireRekorProof = options.RequireRekorProof, + TrustedKeyIds = options.TrustedKeyIds.Count > 0 ? [.. options.TrustedKeyIds] : null, + SharedSecret = options.SignatureSecret, + SharedSecretFile = options.SignatureSecretFile, + VerifyIntegrity = true, + SkipSignatureVerification = !options.RequireSignatureVerification + }; + + _logger.LogDebug("SecretsAnalyzerHost: Verifying bundle signature"); + + var result = await _bundleVerifier.VerifyAsync( + options.RulesetPath, + verificationOptions, + ct); + + LastVerificationResult = result; + + if (!result.IsValid) + { + var errorMessage = $"Bundle verification failed: {string.Join("; ", result.ValidationErrors)}"; + _logger.LogError("SecretsAnalyzerHost: {Error}", errorMessage); + + if (options.FailOnInvalidBundle) + { + throw new InvalidOperationException(errorMessage); + } + + // Allow loading but log prominently + _logger.LogWarning( + "SecretsAnalyzerHost: Continuing with unverified bundle. " + + "Set RequireSignatureVerification=true to enforce verification."); + return; + } + + _logger.LogInformation( + "SecretsAnalyzerHost: Bundle verified - signed by {KeyId} at {SignedAt}", + result.SignerKeyId ?? "unknown", + result.SignedAt?.ToString("o") ?? "unknown"); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerOptions.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerOptions.cs new file mode 100644 index 000000000..b6049886f --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/SecretsAnalyzerOptions.cs @@ -0,0 +1,118 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Configuration options for the secrets analyzer. +/// +public sealed class SecretsAnalyzerOptions +{ + /// + /// Configuration section name. + /// + public const string SectionName = "Scanner:Analyzers:Secrets"; + + /// + /// Enable secret leak detection (experimental feature). + /// Default: false (opt-in). + /// + public bool Enabled { get; set; } = false; + + /// + /// Path to the ruleset bundle directory. + /// + [Required] + public string RulesetPath { get; set; } = "/opt/stellaops/plugins/scanner/analyzers/secrets"; + + /// + /// Minimum confidence level to report findings. + /// + public SecretConfidence MinConfidence { get; set; } = SecretConfidence.Low; + + /// + /// Maximum findings per scan (circuit breaker). + /// + [Range(1, 10000)] + public int MaxFindingsPerScan { get; set; } = 1000; + + /// + /// Maximum file size to scan in bytes. + /// Files larger than this are skipped. + /// + [Range(1, 100 * 1024 * 1024)] + public long MaxFileSizeBytes { get; set; } = 10 * 1024 * 1024; // 10MB + + /// + /// Enable entropy-based detection. + /// + public bool EnableEntropyDetection { get; set; } = true; + + /// + /// Default entropy threshold (bits per character). + /// + [Range(3.0, 8.0)] + public double EntropyThreshold { get; set; } = 4.5; + + /// + /// File extensions to scan. Empty means all text files. + /// + public HashSet IncludeExtensions { get; set; } = []; + + /// + /// File extensions to exclude from scanning. + /// + public HashSet ExcludeExtensions { get; set; } = + [ + ".png", ".jpg", ".jpeg", ".gif", ".ico", ".webp", + ".zip", ".tar", ".gz", ".bz2", ".xz", + ".exe", ".dll", ".so", ".dylib", + ".pdf", ".doc", ".docx", ".xls", ".xlsx", + ".mp3", ".mp4", ".avi", ".mov", ".mkv", + ".ttf", ".woff", ".woff2", ".eot", + ".min.js", ".min.css" + ]; + + /// + /// Directories to exclude from scanning (glob patterns). + /// + public HashSet ExcludeDirectories { get; set; } = + [ + "**/node_modules/**", + "**/.git/**", + "**/vendor/**", + "**/__pycache__/**", + "**/bin/**", + "**/obj/**" + ]; + + /// + /// Whether to fail the scan if the bundle cannot be loaded. + /// + public bool FailOnInvalidBundle { get; set; } = false; + + /// + /// Whether to require DSSE signature verification for bundles. + /// + public bool RequireSignatureVerification { get; set; } = false; + + /// + /// Shared secret for HMAC signature verification (base64 or hex). + /// + public string? SignatureSecret { get; set; } + + /// + /// Path to file containing the shared secret. + /// + public string? SignatureSecretFile { get; set; } + + /// + /// List of trusted key IDs for signature verification. + /// If empty, any key is accepted. + /// + public HashSet TrustedKeyIds { get; set; } = []; + + /// + /// Whether to require Rekor transparency log proof. + /// + public bool RequireRekorProof { get; set; } = false; +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/ServiceCollectionExtensions.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/ServiceCollectionExtensions.cs new file mode 100644 index 000000000..9b10f55fd --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/ServiceCollectionExtensions.cs @@ -0,0 +1,90 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Scanner.Analyzers.Lang; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; + +namespace StellaOps.Scanner.Analyzers.Secrets; + +/// +/// Extension methods for registering secrets analyzer services. +/// +public static class ServiceCollectionExtensions +{ + /// + /// Adds the secrets analyzer services to the service collection. + /// + /// The service collection. + /// The configuration. + /// The service collection for chaining. + public static IServiceCollection AddSecretsAnalyzer( + this IServiceCollection services, + IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + // Register options + services.AddOptions() + .Bind(configuration.GetSection(SecretsAnalyzerOptions.SectionName)) + .ValidateDataAnnotations() + .ValidateOnStart(); + + // Register TimeProvider if not already registered + services.TryAddSingleton(TimeProvider.System); + + RegisterCoreServices(services); + return services; + } + + /// + /// Adds the secrets analyzer services with custom options. + /// + /// The service collection. + /// Action to configure options. + /// The service collection for chaining. + public static IServiceCollection AddSecretsAnalyzer( + this IServiceCollection services, + Action configureOptions) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureOptions); + + // Register options + services.AddOptions() + .Configure(configureOptions) + .ValidateDataAnnotations() + .ValidateOnStart(); + + RegisterCoreServices(services); + return services; + } + + private static void RegisterCoreServices(IServiceCollection services) + { + // Register TimeProvider if not already registered + services.TryAddSingleton(TimeProvider.System); + + // Register core services + services.AddSingleton(); + services.AddSingleton(); + + // Register detectors + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + // Register bundle infrastructure (Sprint: SPRINT_20260104_003_SCANNER) + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + // Register analyzer + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + + // Register hosted service + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/StellaOps.Scanner.Analyzers.Secrets.csproj b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/StellaOps.Scanner.Analyzers.Secrets.csproj new file mode 100644 index 000000000..d9674760c --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Secrets/StellaOps.Scanner.Analyzers.Secrets.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.CallGraph/DependencyInjection/CallGraphServiceCollectionExtensions.cs b/src/Scanner/__Libraries/StellaOps.Scanner.CallGraph/DependencyInjection/CallGraphServiceCollectionExtensions.cs index 25095072a..15fa946dd 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.CallGraph/DependencyInjection/CallGraphServiceCollectionExtensions.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.CallGraph/DependencyInjection/CallGraphServiceCollectionExtensions.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using StellaOps.Scanner.CallGraph.Binary; using StellaOps.Scanner.CallGraph.Caching; using StellaOps.Scanner.CallGraph.DotNet; using StellaOps.Scanner.CallGraph.Go; @@ -40,6 +41,7 @@ public static class CallGraphServiceCollectionExtensions services.AddSingleton(); // Node.js/JavaScript via Babel services.AddSingleton(); // Python via AST analysis services.AddSingleton(); // Go via SSA analysis + services.AddSingleton(); // Native ELF/PE/Mach-O binaries // Register the extractor registry for language-based lookup services.AddSingleton(); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs index c47f406d7..d4147676f 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs @@ -50,4 +50,8 @@ public static class ScanAnalysisKeys public const string VulnerabilityMatches = "analysis.poe.vulnerability.matches"; public const string PoEResults = "analysis.poe.results"; public const string PoEConfiguration = "analysis.poe.configuration"; + + // Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection + public const string SecretFindings = "analysis.secrets.findings"; + public const string SecretRulesetVersion = "analysis.secrets.ruleset.version"; } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleBuilderTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleBuilderTests.cs new file mode 100644 index 000000000..6234511e7 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleBuilderTests.cs @@ -0,0 +1,264 @@ +// ----------------------------------------------------------------------------- +// BundleBuilderTests.cs +// Sprint: SPRINT_20260104_003_SCANNER (Secret Detection Rule Bundles) +// Task: RB-011 - Unit tests for bundle building. +// ----------------------------------------------------------------------------- + +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Analyzers.Secrets; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests.Bundles; + +[Trait("Category", "Unit")] +public class BundleBuilderTests : IDisposable +{ + private readonly string _tempDir; + private readonly string _sourceDir; + private readonly string _outputDir; + private readonly BundleBuilder _sut; + private readonly FakeTimeProvider _timeProvider; + + public BundleBuilderTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"bundle-test-{Guid.NewGuid():N}"); + _sourceDir = Path.Combine(_tempDir, "sources"); + _outputDir = Path.Combine(_tempDir, "output"); + + Directory.CreateDirectory(_sourceDir); + Directory.CreateDirectory(_outputDir); + + var validator = new RuleValidator(NullLogger.Instance); + _sut = new BundleBuilder(validator, NullLogger.Instance); + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + [Fact] + public async Task BuildAsync_ValidRules_CreatesBundle() + { + // Arrange + var rule1Path = CreateRuleFile("rule1.json", new SecretRule + { + Id = "stellaops.secrets.test-rule1", + Version = "1.0.0", + Name = "Test Rule 1", + Description = "A test rule for validation", + Type = SecretRuleType.Regex, + Pattern = "[A-Z]{10}", + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Enabled = true + }); + + var rule2Path = CreateRuleFile("rule2.json", new SecretRule + { + Id = "stellaops.secrets.test-rule2", + Version = "1.0.0", + Name = "Test Rule 2", + Description = "Another test rule", + Type = SecretRuleType.Regex, + Pattern = "[0-9]{8}", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium, + Enabled = true + }); + + var options = new BundleBuildOptions + { + RuleFiles = new[] { rule1Path, rule2Path }, + OutputDirectory = _outputDir, + BundleId = "test-bundle", + Version = "2026.01", + TimeProvider = _timeProvider + }; + + // Act + var artifact = await _sut.BuildAsync(options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(artifact); + Assert.Equal("test-bundle", artifact.Manifest.Id); + Assert.Equal("2026.01", artifact.Manifest.Version); + Assert.Equal(2, artifact.TotalRules); + Assert.Equal(2, artifact.EnabledRules); + Assert.True(File.Exists(artifact.ManifestPath)); + Assert.True(File.Exists(artifact.RulesPath)); + } + + [Fact] + public async Task BuildAsync_SortsRulesById() + { + // Arrange + var zebraPath = CreateRuleFile("z-rule.json", new SecretRule + { + Id = "stellaops.secrets.zebra", + Version = "1.0.0", + Name = "Zebra Rule", + Description = "Rule that should sort last", + Type = SecretRuleType.Regex, + Pattern = "zebra", + Severity = SecretSeverity.Low, + Confidence = SecretConfidence.Medium, + Enabled = true + }); + + var alphaPath = CreateRuleFile("a-rule.json", new SecretRule + { + Id = "stellaops.secrets.alpha", + Version = "1.0.0", + Name = "Alpha Rule", + Description = "Rule that should sort first", + Type = SecretRuleType.Regex, + Pattern = "alpha", + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Enabled = true + }); + + var options = new BundleBuildOptions + { + RuleFiles = new[] { zebraPath, alphaPath }, + OutputDirectory = _outputDir, + BundleId = "sorted-bundle", + Version = "1.0.0", + TimeProvider = _timeProvider + }; + + // Act + var artifact = await _sut.BuildAsync(options, TestContext.Current.CancellationToken); + + // Assert - check the manifest rules array is sorted (manifest is already built) + Assert.Equal(2, artifact.Manifest.Rules.Length); + Assert.Equal("stellaops.secrets.alpha", artifact.Manifest.Rules[0].Id); + Assert.Equal("stellaops.secrets.zebra", artifact.Manifest.Rules[1].Id); + } + + [Fact] + public async Task BuildAsync_ComputesCorrectSha256() + { + // Arrange + var rulePath = CreateRuleFile("rule.json", new SecretRule + { + Id = "stellaops.secrets.hash-test", + Version = "1.0.0", + Name = "Hash Test", + Description = "Rule for testing SHA-256 computation", + Type = SecretRuleType.Regex, + Pattern = "test123", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium, + Enabled = true + }); + + var options = new BundleBuildOptions + { + RuleFiles = new[] { rulePath }, + OutputDirectory = _outputDir, + BundleId = "hash-bundle", + Version = "1.0.0", + TimeProvider = _timeProvider + }; + + // Act + var artifact = await _sut.BuildAsync(options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotEmpty(artifact.RulesSha256); + Assert.Matches("^[a-f0-9]{64}$", artifact.RulesSha256); + + // Verify hash matches file content + await using var stream = File.OpenRead(artifact.RulesPath); + var hash = await System.Security.Cryptography.SHA256.HashDataAsync(stream, TestContext.Current.CancellationToken); + var expectedHash = Convert.ToHexString(hash).ToLowerInvariant(); + Assert.Equal(expectedHash, artifact.RulesSha256); + } + + [Fact] + public async Task BuildAsync_InvalidRule_ThrowsException() + { + // Arrange - create an invalid rule (id not properly namespaced) + var invalidRulePath = Path.Combine(_sourceDir, "invalid-rule.json"); + var invalidRuleJson = JsonSerializer.Serialize(new + { + id = "invalid", // Not namespaced with stellaops.secrets + version = "1.0.0", + name = "Invalid Rule", + description = "This rule has an invalid ID", + type = "regex", + pattern = "test", + severity = "medium", + confidence = "medium" + }, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(invalidRulePath, invalidRuleJson, TestContext.Current.CancellationToken); + + var options = new BundleBuildOptions + { + RuleFiles = new[] { invalidRulePath }, + OutputDirectory = _outputDir, + BundleId = "invalid-bundle", + Version = "1.0.0", + TimeProvider = _timeProvider + }; + + // Act & Assert + await Assert.ThrowsAsync( + () => _sut.BuildAsync(options, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task BuildAsync_EmptyRuleFiles_ThrowsException() + { + // Arrange + var options = new BundleBuildOptions + { + RuleFiles = Array.Empty(), + OutputDirectory = _outputDir, + BundleId = "empty-bundle", + Version = "1.0.0" + }; + + // Act & Assert + await Assert.ThrowsAsync( + () => _sut.BuildAsync(options, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task BuildAsync_NonexistentRuleFile_ThrowsException() + { + // Arrange + var options = new BundleBuildOptions + { + RuleFiles = new[] { Path.Combine(_tempDir, "nonexistent.json") }, + OutputDirectory = _outputDir, + BundleId = "missing-bundle", + Version = "1.0.0" + }; + + // Act & Assert + await Assert.ThrowsAsync( + () => _sut.BuildAsync(options, TestContext.Current.CancellationToken)); + } + + private string CreateRuleFile(string filename, SecretRule rule) + { + var json = JsonSerializer.Serialize(rule, new JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + var path = Path.Combine(_sourceDir, filename); + File.WriteAllText(path, json); + return path; + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleSignerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleSignerTests.cs new file mode 100644 index 000000000..876af2941 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleSignerTests.cs @@ -0,0 +1,217 @@ +// ----------------------------------------------------------------------------- +// BundleSignerTests.cs +// Sprint: SPRINT_20260104_003_SCANNER (Secret Detection Rule Bundles) +// Task: RB-011 - Unit tests for bundle signing. +// ----------------------------------------------------------------------------- + +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Analyzers.Secrets; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests.Bundles; + +[Trait("Category", "Unit")] +public class BundleSignerTests : IDisposable +{ + private readonly string _tempDir; + private readonly BundleSigner _sut; + private readonly FakeTimeProvider _timeProvider; + + public BundleSignerTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"signer-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + + _sut = new BundleSigner(NullLogger.Instance); + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + [Fact] + public async Task SignAsync_ValidArtifact_CreatesDsseEnvelope() + { + // Arrange + var artifact = CreateTestArtifact(); + var options = new BundleSigningOptions + { + KeyId = "test-key-001", + SharedSecret = Convert.ToBase64String(new byte[32]), // 256-bit key + TimeProvider = _timeProvider + }; + + // Act + var result = await _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.True(File.Exists(result.EnvelopePath)); + Assert.NotNull(result.Envelope); + Assert.Single(result.Envelope.Signatures); + Assert.Equal("test-key-001", result.Envelope.Signatures[0].KeyId); + } + + [Fact] + public async Task SignAsync_UpdatesManifestWithSignatureInfo() + { + // Arrange + var artifact = CreateTestArtifact(); + var options = new BundleSigningOptions + { + KeyId = "signer-key", + SharedSecret = Convert.ToBase64String(new byte[32]), + TimeProvider = _timeProvider + }; + + // Act + var result = await _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result.UpdatedManifest.Signatures); + Assert.Equal("signer-key", result.UpdatedManifest.Signatures.KeyId); + Assert.Equal("secrets.ruleset.dsse.json", result.UpdatedManifest.Signatures.DsseEnvelope); + Assert.Equal(_timeProvider.GetUtcNow(), result.UpdatedManifest.Signatures.SignedAt); + } + + [Fact] + public async Task SignAsync_EnvelopeContainsBase64UrlPayload() + { + // Arrange + var artifact = CreateTestArtifact(); + var options = new BundleSigningOptions + { + KeyId = "test-key", + SharedSecret = Convert.ToBase64String(new byte[32]), + TimeProvider = _timeProvider + }; + + // Act + var result = await _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotEmpty(result.Envelope.Payload); + // Base64url should not contain +, /, or = + Assert.DoesNotContain("+", result.Envelope.Payload); + Assert.DoesNotContain("/", result.Envelope.Payload); + Assert.DoesNotContain("=", result.Envelope.Payload); + } + + [Fact] + public async Task SignAsync_WithSecretFile_LoadsSecret() + { + // Arrange + var artifact = CreateTestArtifact(); + var secretFile = Path.Combine(_tempDir, "secret.key"); + var secret = Convert.ToBase64String(new byte[32]); + await File.WriteAllTextAsync(secretFile, secret, TestContext.Current.CancellationToken); + + var options = new BundleSigningOptions + { + KeyId = "file-key", + SharedSecretFile = secretFile, + TimeProvider = _timeProvider + }; + + // Act + var result = await _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Envelope); + } + + [Fact] + public async Task SignAsync_WithoutSecret_ThrowsException() + { + // Arrange + var artifact = CreateTestArtifact(); + var options = new BundleSigningOptions + { + KeyId = "no-secret-key", + TimeProvider = _timeProvider + }; + + // Act & Assert + await Assert.ThrowsAsync( + () => _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task SignAsync_UnsupportedAlgorithm_ThrowsException() + { + // Arrange + var artifact = CreateTestArtifact(); + var options = new BundleSigningOptions + { + KeyId = "test-key", + SharedSecret = Convert.ToBase64String(new byte[32]), + Algorithm = "ES256", // Not supported + TimeProvider = _timeProvider + }; + + // Act & Assert + await Assert.ThrowsAsync( + () => _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task SignAsync_HexEncodedSecret_Works() + { + // Arrange + var artifact = CreateTestArtifact(); + var hexSecret = new string('a', 64); // 32 bytes as hex + var options = new BundleSigningOptions + { + KeyId = "hex-key", + SharedSecret = hexSecret, + TimeProvider = _timeProvider + }; + + // Act + var result = await _sut.SignAsync(artifact, options, TestContext.Current.CancellationToken); + + // Assert + Assert.NotNull(result); + } + + private BundleArtifact CreateTestArtifact() + { + var manifest = new BundleManifest + { + Id = "test-bundle", + Version = "1.0.0", + CreatedAt = _timeProvider.GetUtcNow(), + Integrity = new BundleIntegrity + { + RulesSha256 = new string('0', 64), + TotalRules = 1, + EnabledRules = 1 + } + }; + + var manifestPath = Path.Combine(_tempDir, "secrets.ruleset.manifest.json"); + var rulesPath = Path.Combine(_tempDir, "secrets.ruleset.rules.jsonl"); + + File.WriteAllText(manifestPath, JsonSerializer.Serialize(manifest, new JsonSerializerOptions { WriteIndented = true })); + File.WriteAllText(rulesPath, "{\"id\":\"test.rule\"}"); + + return new BundleArtifact + { + ManifestPath = manifestPath, + RulesPath = rulesPath, + RulesSha256 = manifest.Integrity.RulesSha256, + TotalRules = 1, + EnabledRules = 1, + Manifest = manifest + }; + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleVerifierTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleVerifierTests.cs new file mode 100644 index 000000000..f59cf8a6e --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/BundleVerifierTests.cs @@ -0,0 +1,328 @@ +// ----------------------------------------------------------------------------- +// BundleVerifierTests.cs +// Sprint: SPRINT_20260104_003_SCANNER (Secret Detection Rule Bundles) +// Task: RB-011 - Unit tests for bundle verification. +// ----------------------------------------------------------------------------- + +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Analyzers.Secrets; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests.Bundles; + +[Trait("Category", "Unit")] +public class BundleVerifierTests : IDisposable +{ + private readonly string _tempDir; + private readonly BundleVerifier _sut; + private readonly FakeTimeProvider _timeProvider; + private readonly byte[] _testSecret; + + public BundleVerifierTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), $"verifier-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_tempDir); + + _sut = new BundleVerifier(NullLogger.Instance); + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + _testSecret = new byte[32]; + RandomNumberGenerator.Fill(_testSecret); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + Directory.Delete(_tempDir, recursive: true); + } + } + + [Fact] + public async Task VerifyAsync_ValidBundle_ReturnsValid() + { + // Arrange + var bundleDir = await CreateSignedBundleAsync(TestContext.Current.CancellationToken); + var options = new BundleVerificationOptions + { + SharedSecret = Convert.ToBase64String(_testSecret), + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.True(result.IsValid); + Assert.Equal("test-bundle", result.BundleId); + Assert.Equal("1.0.0", result.BundleVersion); + Assert.Empty(result.ValidationErrors); + } + + [Fact] + public async Task VerifyAsync_TamperedRulesFile_ReturnsInvalid() + { + // Arrange + var bundleDir = await CreateSignedBundleAsync(TestContext.Current.CancellationToken); + + // Tamper with the rules file + var rulesPath = Path.Combine(bundleDir, "secrets.ruleset.rules.jsonl"); + await File.AppendAllTextAsync(rulesPath, "\n{\"id\":\"injected.rule\"}", TestContext.Current.CancellationToken); + + var options = new BundleVerificationOptions + { + SharedSecret = Convert.ToBase64String(_testSecret), + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.ValidationErrors, e => e.Contains("integrity", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task VerifyAsync_WrongSecret_ReturnsInvalid() + { + // Arrange + var bundleDir = await CreateSignedBundleAsync(TestContext.Current.CancellationToken); + var wrongSecret = new byte[32]; + RandomNumberGenerator.Fill(wrongSecret); + + var options = new BundleVerificationOptions + { + SharedSecret = Convert.ToBase64String(wrongSecret), + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.ValidationErrors, e => e.Contains("signature", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task VerifyAsync_MissingManifest_ReturnsInvalid() + { + // Arrange + var bundleDir = Path.Combine(_tempDir, "missing-manifest"); + Directory.CreateDirectory(bundleDir); + + var options = new BundleVerificationOptions + { + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.ValidationErrors, e => e.Contains("manifest", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task VerifyAsync_NonexistentDirectory_ReturnsInvalid() + { + // Arrange + var options = new BundleVerificationOptions(); + + // Act + var result = await _sut.VerifyAsync(Path.Combine(_tempDir, "nonexistent"), options, TestContext.Current.CancellationToken); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.ValidationErrors, e => e.Contains("not found", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task VerifyAsync_SkipSignatureVerification_OnlyChecksIntegrity() + { + // Arrange + var bundleDir = await CreateUnsignedBundleAsync(TestContext.Current.CancellationToken); + + var options = new BundleVerificationOptions + { + SkipSignatureVerification = true, + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.True(result.IsValid); + } + + [Fact] + public async Task VerifyAsync_UntrustedKeyId_ReturnsInvalid() + { + // Arrange + var bundleDir = await CreateSignedBundleAsync(TestContext.Current.CancellationToken); + + var options = new BundleVerificationOptions + { + SharedSecret = Convert.ToBase64String(_testSecret), + TrustedKeyIds = new[] { "other-trusted-key" }, + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.ValidationErrors, e => e.Contains("trusted", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task VerifyAsync_TrustedKeyId_ReturnsValid() + { + // Arrange + var bundleDir = await CreateSignedBundleAsync(TestContext.Current.CancellationToken); + + var options = new BundleVerificationOptions + { + SharedSecret = Convert.ToBase64String(_testSecret), + TrustedKeyIds = new[] { "test-key-001" }, + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert + Assert.True(result.IsValid); + Assert.Equal("test-key-001", result.SignerKeyId); + } + + [Fact] + public async Task VerifyAsync_RequireRekorProof_ReturnsWarningWhenNotVerified() + { + // Arrange + var bundleDir = await CreateSignedBundleWithRekorAsync(TestContext.Current.CancellationToken); + + var options = new BundleVerificationOptions + { + SharedSecret = Convert.ToBase64String(_testSecret), + RequireRekorProof = true, + VerifyIntegrity = true + }; + + // Act + var result = await _sut.VerifyAsync(bundleDir, options, TestContext.Current.CancellationToken); + + // Assert (Rekor verification not implemented, should have warning) + Assert.NotEmpty(result.ValidationWarnings); + Assert.Contains(result.ValidationWarnings, w => w.Contains("Rekor", StringComparison.OrdinalIgnoreCase)); + } + + private async Task CreateUnsignedBundleAsync(CancellationToken ct = default) + { + var bundleDir = Path.Combine(_tempDir, $"bundle-{Guid.NewGuid():N}"); + Directory.CreateDirectory(bundleDir); + + // Create rules file + var rulesPath = Path.Combine(bundleDir, "secrets.ruleset.rules.jsonl"); + var ruleJson = JsonSerializer.Serialize(new SecretRule + { + Id = "stellaops.secrets.test-rule", + Version = "1.0.0", + Name = "Test", + Description = "A test rule for verification tests", + Type = SecretRuleType.Regex, + Pattern = "test", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium + }); + await File.WriteAllTextAsync(rulesPath, ruleJson, ct); + + // Compute hash + await using var stream = File.OpenRead(rulesPath); + var hash = await SHA256.HashDataAsync(stream, ct); + var hashHex = Convert.ToHexString(hash).ToLowerInvariant(); + + // Create manifest + var manifest = new BundleManifest + { + Id = "test-bundle", + Version = "1.0.0", + CreatedAt = _timeProvider.GetUtcNow(), + Integrity = new BundleIntegrity + { + RulesSha256 = hashHex, + TotalRules = 1, + EnabledRules = 1 + } + }; + + var manifestPath = Path.Combine(bundleDir, "secrets.ruleset.manifest.json"); + await File.WriteAllTextAsync(manifestPath, + JsonSerializer.Serialize(manifest, new JsonSerializerOptions { WriteIndented = true }), ct); + + return bundleDir; + } + + private async Task CreateSignedBundleAsync(CancellationToken ct = default) + { + var bundleDir = await CreateUnsignedBundleAsync(ct); + + // Sign the bundle + var signer = new BundleSigner(NullLogger.Instance); + var manifestPath = Path.Combine(bundleDir, "secrets.ruleset.manifest.json"); + var manifestJson = await File.ReadAllTextAsync(manifestPath, ct); + var manifest = JsonSerializer.Deserialize(manifestJson, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; + + var artifact = new BundleArtifact + { + ManifestPath = manifestPath, + RulesPath = Path.Combine(bundleDir, "secrets.ruleset.rules.jsonl"), + RulesSha256 = manifest.Integrity.RulesSha256, + TotalRules = 1, + EnabledRules = 1, + Manifest = manifest + }; + + await signer.SignAsync(artifact, new BundleSigningOptions + { + KeyId = "test-key-001", + SharedSecret = Convert.ToBase64String(_testSecret), + TimeProvider = _timeProvider + }, ct); + + return bundleDir; + } + + private async Task CreateSignedBundleWithRekorAsync(CancellationToken ct = default) + { + var bundleDir = await CreateSignedBundleAsync(ct); + + // Update manifest to include Rekor log ID + var manifestPath = Path.Combine(bundleDir, "secrets.ruleset.manifest.json"); + var manifestJson = await File.ReadAllTextAsync(manifestPath, ct); + var manifest = JsonSerializer.Deserialize(manifestJson, + new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!; + + var updatedManifest = manifest with + { + Signatures = manifest.Signatures! with + { + RekorLogId = "rekor-log-entry-123456" + } + }; + + await File.WriteAllTextAsync(manifestPath, + JsonSerializer.Serialize(updatedManifest, new JsonSerializerOptions { WriteIndented = true }), ct); + + return bundleDir; + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/RuleValidatorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/RuleValidatorTests.cs new file mode 100644 index 000000000..9e1081a8c --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/Bundles/RuleValidatorTests.cs @@ -0,0 +1,228 @@ +// ----------------------------------------------------------------------------- +// RuleValidatorTests.cs +// Sprint: SPRINT_20260104_003_SCANNER (Secret Detection Rule Bundles) +// Task: RB-011 - Unit tests for rule validation. +// ----------------------------------------------------------------------------- + +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scanner.Analyzers.Secrets; +using StellaOps.Scanner.Analyzers.Secrets.Bundles; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests.Bundles; + +[Trait("Category", "Unit")] +public class RuleValidatorTests +{ + private readonly RuleValidator _sut; + + public RuleValidatorTests() + { + _sut = new RuleValidator(NullLogger.Instance); + } + + [Fact] + public void Validate_ValidRule_ReturnsValid() + { + // Arrange + var rule = new SecretRule + { + Id = "stellaops.secrets.test-rule", + Version = "1.0.0", + Name = "Test Rule", + Description = "A test rule for validation", + Type = SecretRuleType.Regex, + Pattern = "[A-Z]{10}", + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High + }; + + // Act + var result = _sut.Validate(rule); + + // Assert + Assert.True(result.IsValid); + Assert.Empty(result.Errors); + } + + [Theory] + [InlineData("")] + [InlineData("invalid-id")] // No namespace separator (no dots) + [InlineData("InvalidCase.rule")] // Starts with uppercase + public void Validate_InvalidId_ReturnsError(string invalidId) + { + // Arrange + var rule = new SecretRule + { + Id = invalidId, + Version = "1.0.0", + Name = "Test Rule", + Description = "Test description", + Type = SecretRuleType.Regex, + Pattern = "[A-Z]{10}", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium + }; + + // Act + var result = _sut.Validate(rule); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.Contains("ID")); + } + + [Theory] + [InlineData("")] + [InlineData("invalid")] + [InlineData("1.0")] + [InlineData("v1.0.0")] + public void Validate_InvalidVersion_ReturnsError(string invalidVersion) + { + // Arrange + var rule = new SecretRule + { + Id = "stellaops.secrets.test", + Version = invalidVersion, + Name = "Test Rule", + Description = "Test description", + Type = SecretRuleType.Regex, + Pattern = "[A-Z]{10}", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium + }; + + // Act + var result = _sut.Validate(rule); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.Contains("version", StringComparison.OrdinalIgnoreCase)); + } + + [Theory] + [InlineData("[")] + [InlineData("(unclosed")] + [InlineData("(?invalid)")] + public void Validate_InvalidRegex_ReturnsError(string invalidPattern) + { + // Arrange + var rule = new SecretRule + { + Id = "stellaops.secrets.test", + Version = "1.0.0", + Name = "Test Rule", + Description = "Test description", + Type = SecretRuleType.Regex, + Pattern = invalidPattern, + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium + }; + + // Act + var result = _sut.Validate(rule); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.Contains("regex", StringComparison.OrdinalIgnoreCase) || + e.Contains("pattern", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_EmptyPattern_ReturnsError() + { + // Arrange + var rule = new SecretRule + { + Id = "stellaops.secrets.test", + Version = "1.0.0", + Name = "Test Rule", + Description = "Test description", + Type = SecretRuleType.Regex, + Pattern = "", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium + }; + + // Act + var result = _sut.Validate(rule); + + // Assert + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.Contains("pattern", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_ValidEntropyRule_ReturnsValid() + { + // Arrange + var rule = new SecretRule + { + Id = "stellaops.secrets.entropy-test", + Version = "1.0.0", + Name = "Entropy Test", + Description = "Detects high-entropy strings", + Type = SecretRuleType.Entropy, + Pattern = "", // Pattern can be empty for entropy rules + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium, + EntropyThreshold = 4.5 + }; + + // Act + var result = _sut.Validate(rule); + + // Assert + Assert.True(result.IsValid); + } + + [Fact] + public void Validate_EntropyRuleWithDefaultThreshold_ReturnsValid() + { + // Arrange - using the default entropy threshold (4.5) which is in valid range + var rule = new SecretRule + { + Id = "stellaops.secrets.entropy-test", + Version = "1.0.0", + Name = "Entropy Test", + Description = "Detects high-entropy strings with default threshold", + Type = SecretRuleType.Entropy, + Pattern = "", // Pattern can be empty for entropy rules + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium + // Default entropy threshold is 4.5, which is in the valid range + }; + + // Act + var result = _sut.Validate(rule); + + // Assert - default threshold (4.5) is valid, no warnings expected + Assert.True(result.IsValid); + Assert.Empty(result.Warnings); + } + + [Fact] + public void Validate_EntropyRuleWithOutOfRangeThreshold_ReturnsWarning() + { + // Arrange - using an out-of-range threshold + var rule = new SecretRule + { + Id = "stellaops.secrets.entropy-test", + Version = "1.0.0", + Name = "Entropy Test", + Description = "Detects high-entropy strings with extreme threshold", + Type = SecretRuleType.Entropy, + Pattern = "", + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium, + EntropyThreshold = 0 // Zero triggers <= 0 warning condition + }; + + // Act + var result = _sut.Validate(rule); + + // Assert - valid but with warning about unusual threshold + Assert.True(result.IsValid); + Assert.Contains(result.Warnings, w => w.Contains("entropy", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/EntropyCalculatorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/EntropyCalculatorTests.cs new file mode 100644 index 000000000..3d219c2cd --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/EntropyCalculatorTests.cs @@ -0,0 +1,110 @@ +using FluentAssertions; +using StellaOps.Scanner.Analyzers.Secrets; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests; + +[Trait("Category", "Unit")] +public sealed class EntropyCalculatorTests +{ + [Fact] + public void Calculate_EmptyString_ReturnsZero() + { + var entropy = EntropyCalculator.Calculate(string.Empty); + + entropy.Should().Be(0.0); + } + + [Fact] + public void Calculate_SingleCharacter_ReturnsZero() + { + var entropy = EntropyCalculator.Calculate("a"); + + entropy.Should().Be(0.0); + } + + [Fact] + public void Calculate_RepeatedCharacter_ReturnsZero() + { + var entropy = EntropyCalculator.Calculate("aaaaaaaaaa"); + + entropy.Should().Be(0.0); + } + + [Fact] + public void Calculate_TwoDistinctCharacters_ReturnsOne() + { + var entropy = EntropyCalculator.Calculate("ababababab"); + + entropy.Should().BeApproximately(1.0, 0.01); + } + + [Fact] + public void Calculate_FourDistinctCharacters_ReturnsTwo() + { + var entropy = EntropyCalculator.Calculate("abcdabcdabcd"); + + entropy.Should().BeApproximately(2.0, 0.01); + } + + [Fact] + public void Calculate_HighEntropyString_ReturnsHighValue() + { + var highEntropyString = "aB1cD2eF3gH4iJ5kL6mN7oP8qR9sT0uV"; + + var entropy = EntropyCalculator.Calculate(highEntropyString); + + entropy.Should().BeGreaterThan(4.0); + } + + [Fact] + public void Calculate_LowEntropyPassword_ReturnsLowValue() + { + var lowEntropyString = "password"; + + var entropy = EntropyCalculator.Calculate(lowEntropyString); + + entropy.Should().BeLessThan(3.0); + } + + [Fact] + public void Calculate_AwsAccessKeyPattern_ReturnsHighEntropy() + { + var awsKey = "AKIAIOSFODNN7EXAMPLE"; + + var entropy = EntropyCalculator.Calculate(awsKey); + + entropy.Should().BeGreaterThan(3.5); + } + + [Fact] + public void Calculate_Base64String_ReturnsHighEntropy() + { + var base64 = "SGVsbG8gV29ybGQhIFRoaXMgaXMgYSB0ZXN0"; + + var entropy = EntropyCalculator.Calculate(base64); + + entropy.Should().BeGreaterThan(4.0); + } + + [Fact] + public void Calculate_IsDeterministic() + { + var input = "TestString123!@#"; + + var entropy1 = EntropyCalculator.Calculate(input); + var entropy2 = EntropyCalculator.Calculate(input); + + entropy1.Should().Be(entropy2); + } + + [Theory] + [InlineData("0123456789", 3.32)] + [InlineData("abcdefghij", 3.32)] + [InlineData("ABCDEFGHIJ", 3.32)] + public void Calculate_KnownPatterns_ReturnsExpectedEntropy(string input, double expectedEntropy) + { + var entropy = EntropyCalculator.Calculate(input); + + entropy.Should().BeApproximately(expectedEntropy, 0.1); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/PayloadMaskerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/PayloadMaskerTests.cs new file mode 100644 index 000000000..e6ea059d6 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/PayloadMaskerTests.cs @@ -0,0 +1,171 @@ +using FluentAssertions; +using StellaOps.Scanner.Analyzers.Secrets; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests; + +[Trait("Category", "Unit")] +public sealed class PayloadMaskerTests +{ + private readonly PayloadMasker _masker = new(); + + [Fact] + public void Mask_EmptySpan_ReturnsEmpty() + { + _masker.Mask(ReadOnlySpan.Empty).Should().BeEmpty(); + } + + [Fact] + public void Mask_ShortValue_ReturnsMaskChars() + { + // Values shorter than prefix+suffix get masked placeholder + var result = _masker.Mask("abc".AsSpan()); + + result.Should().Contain("*"); + } + + [Fact] + public void Mask_StandardValue_PreservesPrefixAndSuffix() + { + var result = _masker.Mask("1234567890".AsSpan()); + + // Default: 4 char prefix, 2 char suffix + result.Should().StartWith("1234"); + result.Should().EndWith("90"); + result.Should().Contain("****"); + } + + [Fact] + public void Mask_AwsAccessKey_PreservesPrefix() + { + var awsKey = "AKIAIOSFODNN7EXAMPLE"; + + var result = _masker.Mask(awsKey.AsSpan()); + + result.Should().StartWith("AKIA"); + result.Should().EndWith("LE"); + result.Should().Contain("****"); + } + + [Fact] + public void Mask_WithPrefixHint_UsesCustomPrefixLength() + { + var apiKey = "sk-proj-abcdefghijklmnop"; + + // MaxExposedChars is 6, so prefix:8 + suffix:2 gets scaled down + var result = _masker.Mask(apiKey.AsSpan(), "prefix:4,suffix:2"); + + result.Should().StartWith("sk-p"); + result.Should().Contain("****"); + } + + [Fact] + public void Mask_LongValue_MasksMiddle() + { + var longSecret = "verylongsecretthatexceeds100characters" + + "andshouldbemaskkedproperlywithoutexpo" + + "singtheentirecontentstoanyoneviewingit"; + + var result = _masker.Mask(longSecret.AsSpan()); + + // Should contain mask characters and be shorter than original + result.Should().Contain("****"); + result.Length.Should().BeLessThan(longSecret.Length); + } + + [Fact] + public void Mask_IsDeterministic() + { + var secret = "AKIAIOSFODNN7EXAMPLE"; + + var result1 = _masker.Mask(secret.AsSpan()); + var result2 = _masker.Mask(secret.AsSpan()); + + result1.Should().Be(result2); + } + + [Fact] + public void Mask_NeverExposesFullSecret() + { + var secret = "supersecretkey123"; + + var result = _masker.Mask(secret.AsSpan()); + + result.Should().NotBe(secret); + result.Should().Contain("*"); + } + + [Theory] + [InlineData("prefix:6,suffix:0")] + [InlineData("prefix:0,suffix:6")] + [InlineData("prefix:3,suffix:3")] + public void Mask_WithVariousHints_RespectsTotalLimit(string hint) + { + var secret = "abcdefghijklmnopqrstuvwxyz"; + + var result = _masker.Mask(secret.AsSpan(), hint); + + var visibleChars = result.Replace("*", "").Length; + visibleChars.Should().BeLessThanOrEqualTo(PayloadMasker.MaxExposedChars); + } + + [Fact] + public void Mask_EnforcesMinOutputLength() + { + var secret = "abcdefghijklmnop"; + + var result = _masker.Mask(secret.AsSpan()); + + result.Length.Should().BeGreaterThanOrEqualTo(PayloadMasker.MinOutputLength); + } + + [Fact] + public void Mask_ByteOverload_DecodesUtf8() + { + var text = "secretpassword123"; + var bytes = System.Text.Encoding.UTF8.GetBytes(text); + + var result = _masker.Mask(bytes.AsSpan()); + + result.Should().Contain("****"); + result.Should().StartWith("secr"); + } + + [Fact] + public void Mask_EmptyByteSpan_ReturnsEmpty() + { + _masker.Mask(ReadOnlySpan.Empty).Should().BeEmpty(); + } + + [Fact] + public void Mask_InvalidHint_UsesDefaults() + { + var secret = "abcdefghijklmnop"; + + var result1 = _masker.Mask(secret.AsSpan(), "invalid:hint:format"); + var result2 = _masker.Mask(secret.AsSpan()); + + result1.Should().Be(result2); + } + + [Fact] + public void Mask_UsesCorrectMaskChar() + { + var secret = "abcdefghijklmnop"; + + var result = _masker.Mask(secret.AsSpan()); + + result.Should().Contain(PayloadMasker.MaskChar.ToString()); + } + + [Fact] + public void Mask_MaskLengthLimited() + { + var longSecret = new string('x', 100); + + var result = _masker.Mask(longSecret.AsSpan()); + + // Count mask characters + var maskCount = result.Count(c => c == PayloadMasker.MaskChar); + maskCount.Should().BeLessThanOrEqualTo(PayloadMasker.MaxMaskLength); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RegexDetectorTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RegexDetectorTests.cs new file mode 100644 index 000000000..ff2150629 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RegexDetectorTests.cs @@ -0,0 +1,235 @@ +using System.Collections.Immutable; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scanner.Analyzers.Secrets; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests; + +[Trait("Category", "Unit")] +public sealed class RegexDetectorTests +{ + private readonly RegexDetector _detector = new(NullLogger.Instance); + + [Fact] + public void DetectorId_ReturnsRegex() + { + _detector.DetectorId.Should().Be("regex"); + } + + [Fact] + public void CanHandle_RegexType_ReturnsTrue() + { + _detector.CanHandle(SecretRuleType.Regex).Should().BeTrue(); + } + + [Fact] + public void CanHandle_EntropyType_ReturnsFalse() + { + _detector.CanHandle(SecretRuleType.Entropy).Should().BeFalse(); + } + + [Fact] + public void CanHandle_CompositeType_ReturnsTrue() + { + // RegexDetector handles both Regex and Composite types + _detector.CanHandle(SecretRuleType.Composite).Should().BeTrue(); + } + + [Fact] + public async Task DetectAsync_NoMatch_ReturnsEmpty() + { + var rule = CreateRule(@"AKIA[0-9A-Z]{16}"); + var content = Encoding.UTF8.GetBytes("no aws key here"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "test.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().BeEmpty(); + } + + [Fact] + public async Task DetectAsync_SingleMatch_ReturnsOne() + { + var rule = CreateRule(@"AKIA[0-9A-Z]{16}"); + var content = Encoding.UTF8.GetBytes("aws_key = AKIAIOSFODNN7EXAMPLE"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "test.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(1); + matches[0].Rule.Id.Should().Be("test-rule"); + } + + [Fact] + public async Task DetectAsync_MultipleMatches_ReturnsAll() + { + var rule = CreateRule(@"AKIA[0-9A-Z]{16}"); + var content = Encoding.UTF8.GetBytes( + "key1 = AKIAIOSFODNN7EXAMPLE\n" + + "key2 = AKIABCDEFGHIJKLMNOP1"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "test.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(2); + } + + [Fact] + public async Task DetectAsync_ReportsCorrectLineNumber() + { + var rule = CreateRule(@"secret_key\s*=\s*\S+"); + var content = Encoding.UTF8.GetBytes( + "# config file\n" + + "debug = true\n" + + "secret_key = mysecretvalue\n" + + "port = 8080"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "config.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(1); + matches[0].LineNumber.Should().Be(3); + } + + [Fact] + public async Task DetectAsync_ReportsCorrectColumn() + { + var rule = CreateRule(@"secret_key"); + var content = Encoding.UTF8.GetBytes("config: secret_key = value"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "test.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(1); + // "secret_key" starts at index 8 (0-based), column 9 (1-based) + matches[0].ColumnStart.Should().Be(9); + } + + [Fact] + public async Task DetectAsync_HandlesMultilineContent() + { + var rule = CreateRule(@"API_KEY\s*=\s*\w+"); + var content = Encoding.UTF8.GetBytes( + "line1\n" + + "line2\n" + + "API_KEY = abc123\n" + + "line4\n" + + "API_KEY = xyz789"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "test.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(2); + matches[0].LineNumber.Should().Be(3); + matches[1].LineNumber.Should().Be(5); + } + + [Fact] + public async Task DetectAsync_DisabledRule_StillProcesses() + { + // Note: The detector doesn't filter by Enabled status. + // Filtering disabled rules is the caller's responsibility (e.g., SecretsAnalyzer) + var rule = CreateRule(@"AKIA[0-9A-Z]{16}", enabled: false); + var content = Encoding.UTF8.GetBytes("AKIAIOSFODNN7EXAMPLE"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "test.txt", + rule, + TestContext.Current.CancellationToken); + + // Detector processes regardless of Enabled flag + matches.Should().HaveCount(1); + } + + [Fact] + public async Task DetectAsync_RespectsCancellation() + { + var rule = CreateRule(@"test"); + var content = Encoding.UTF8.GetBytes("test content"); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + // When cancellation is already requested, detector returns empty (doesn't throw) + var matches = await _detector.DetectAsync(content.AsMemory(), "test.txt", rule, cts.Token); + + matches.Should().BeEmpty(); + } + + [Fact] + public async Task DetectAsync_IncludesFilePath() + { + var rule = CreateRule(@"secret"); + var content = Encoding.UTF8.GetBytes("mysecret"); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "path/to/file.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(1); + matches[0].FilePath.Should().Be("path/to/file.txt"); + } + + [Fact] + public async Task DetectAsync_LargeFile_HandlesEfficiently() + { + var rule = CreateRule(@"SECRET_KEY"); + var lines = Enumerable.Range(0, 10000) + .Select(i => i == 5000 ? "SECRET_KEY = value" : $"line {i}") + .ToArray(); + var content = Encoding.UTF8.GetBytes(string.Join("\n", lines)); + + var matches = await _detector.DetectAsync( + content.AsMemory(), + "large.txt", + rule, + TestContext.Current.CancellationToken); + + matches.Should().HaveCount(1); + matches[0].LineNumber.Should().Be(5001); + } + + private static SecretRule CreateRule(string pattern, bool enabled = true) + { + return new SecretRule + { + Id = "test-rule", + Version = "1.0.0", + Name = "Test Rule", + Description = "Test rule for unit tests", + Type = SecretRuleType.Regex, + Pattern = pattern, + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = ImmutableArray.Empty, + FilePatterns = ImmutableArray.Empty, + Enabled = enabled, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RulesetLoaderTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RulesetLoaderTests.cs new file mode 100644 index 000000000..10ba07119 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/RulesetLoaderTests.cs @@ -0,0 +1,348 @@ +using System.Security.Cryptography; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Analyzers.Secrets; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests; + +[Trait("Category", "Unit")] +public sealed class RulesetLoaderTests : IAsyncLifetime +{ + private readonly string _testDir; + private readonly FakeTimeProvider _timeProvider; + private readonly RulesetLoader _loader; + + public RulesetLoaderTests() + { + _testDir = Path.Combine(Path.GetTempPath(), $"secrets-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_testDir); + + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + _loader = new RulesetLoader(NullLogger.Instance, _timeProvider); + } + + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public ValueTask DisposeAsync() + { + if (Directory.Exists(_testDir)) + { + Directory.Delete(_testDir, recursive: true); + } + return ValueTask.CompletedTask; + } + + [Fact] + public async Task LoadAsync_ValidBundle_LoadsRuleset() + { + await CreateValidBundleAsync(TestContext.Current.CancellationToken); + + var ruleset = await _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken); + + ruleset.Should().NotBeNull(); + ruleset.Id.Should().Be("test-secrets"); + ruleset.Version.Should().Be("1.0.0"); + ruleset.Rules.Should().HaveCount(2); + } + + [Fact] + public async Task LoadAsync_MissingDirectory_ThrowsDirectoryNotFound() + { + var nonExistentPath = Path.Combine(_testDir, "does-not-exist"); + + await Assert.ThrowsAsync( + () => _loader.LoadAsync(nonExistentPath, TestContext.Current.CancellationToken).AsTask()); + } + + [Fact] + public async Task LoadAsync_MissingManifest_ThrowsFileNotFound() + { + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.rules.jsonl"), + """{"id":"rule1","pattern":"test"}""", + TestContext.Current.CancellationToken); + + await Assert.ThrowsAsync( + () => _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken).AsTask()); + } + + [Fact] + public async Task LoadAsync_MissingRulesFile_ThrowsFileNotFound() + { + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.manifest.json"), + """{"id":"test","version":"1.0.0"}""", + TestContext.Current.CancellationToken); + + await Assert.ThrowsAsync( + () => _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken).AsTask()); + } + + [Fact] + public async Task LoadAsync_InvalidIntegrity_ThrowsException() + { + await CreateBundleWithBadIntegrityAsync(TestContext.Current.CancellationToken); + + await Assert.ThrowsAsync( + () => _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken).AsTask()); + } + + [Fact] + public async Task LoadAsync_SortsRulesById() + { + await CreateBundleWithUnorderedRulesAsync(TestContext.Current.CancellationToken); + + var ruleset = await _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken); + + ruleset.Rules.Select(r => r.Id).Should().BeInAscendingOrder(); + } + + [Fact] + public async Task LoadAsync_SkipsBlankLines() + { + await CreateBundleWithBlankLinesAsync(TestContext.Current.CancellationToken); + + var ruleset = await _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken); + + ruleset.Rules.Should().HaveCount(2); + } + + [Fact] + public async Task LoadAsync_SkipsInvalidJsonLines() + { + await CreateBundleWithInvalidJsonAsync(TestContext.Current.CancellationToken); + + var ruleset = await _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken); + + // JSONL processes each line independently - invalid lines are skipped but don't stop processing + // So we get rule1 and rule2 (2 rules), with the invalid line skipped + ruleset.Rules.Should().HaveCount(2); + } + + [Fact] + public async Task LoadAsync_SetsCreatedAt() + { + await CreateValidBundleAsync(TestContext.Current.CancellationToken); + + var ruleset = await _loader.LoadAsync(_testDir, TestContext.Current.CancellationToken); + + ruleset.CreatedAt.Should().Be(_timeProvider.GetUtcNow()); + } + + [Fact] + public async Task LoadFromJsonlAsync_ValidStream_LoadsRules() + { + var jsonl = """ + {"id":"rule1","version":"1.0","name":"Rule 1","type":"regex","pattern":"secret","severity":"high","confidence":"high"} + {"id":"rule2","version":"1.0","name":"Rule 2","type":"regex","pattern":"password","severity":"medium","confidence":"medium"} + """; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonl)); + + var ruleset = await _loader.LoadFromJsonlAsync( + stream, + "test-bundle", + "1.0.0", + TestContext.Current.CancellationToken); + + ruleset.Should().NotBeNull(); + ruleset.Id.Should().Be("test-bundle"); + ruleset.Version.Should().Be("1.0.0"); + ruleset.Rules.Should().HaveCount(2); + } + + [Fact] + public async Task LoadFromJsonlAsync_DefaultValues_AppliedCorrectly() + { + var jsonl = """{"id":"minimal-rule","pattern":"test"}"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonl)); + + var ruleset = await _loader.LoadFromJsonlAsync( + stream, + "test", + "1.0", + TestContext.Current.CancellationToken); + + var rule = ruleset.Rules[0]; + rule.Version.Should().Be("1.0.0"); + rule.Enabled.Should().BeTrue(); + rule.Severity.Should().Be(SecretSeverity.Medium); + rule.Confidence.Should().Be(SecretConfidence.Medium); + rule.Type.Should().Be(SecretRuleType.Regex); + rule.EntropyThreshold.Should().Be(4.5); + rule.MinLength.Should().Be(16); + rule.MaxLength.Should().Be(1000); + } + + [Theory] + [InlineData("regex", SecretRuleType.Regex)] + [InlineData("entropy", SecretRuleType.Entropy)] + [InlineData("composite", SecretRuleType.Composite)] + [InlineData("REGEX", SecretRuleType.Regex)] + [InlineData("unknown", SecretRuleType.Regex)] + public async Task LoadFromJsonlAsync_ParsesRuleType(string typeString, SecretRuleType expected) + { + var jsonl = $$$"""{"id":"rule1","pattern":"test","type":"{{{typeString}}}"}"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonl)); + + var ruleset = await _loader.LoadFromJsonlAsync( + stream, + "test", + "1.0", + TestContext.Current.CancellationToken); + + ruleset.Rules[0].Type.Should().Be(expected); + } + + [Theory] + [InlineData("low", SecretSeverity.Low)] + [InlineData("medium", SecretSeverity.Medium)] + [InlineData("high", SecretSeverity.High)] + [InlineData("critical", SecretSeverity.Critical)] + [InlineData("HIGH", SecretSeverity.High)] + public async Task LoadFromJsonlAsync_ParsesSeverity(string severityString, SecretSeverity expected) + { + var jsonl = $$$"""{"id":"rule1","pattern":"test","severity":"{{{severityString}}}"}"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonl)); + + var ruleset = await _loader.LoadFromJsonlAsync( + stream, + "test", + "1.0", + TestContext.Current.CancellationToken); + + ruleset.Rules[0].Severity.Should().Be(expected); + } + + [Fact] + public async Task LoadFromJsonlAsync_ParsesKeywords() + { + var jsonl = """{"id":"rule1","pattern":"test","keywords":["aws","key","secret"]}"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonl)); + + var ruleset = await _loader.LoadFromJsonlAsync( + stream, + "test", + "1.0", + TestContext.Current.CancellationToken); + + ruleset.Rules[0].Keywords.Should().BeEquivalentTo(["aws", "key", "secret"]); + } + + [Fact] + public async Task LoadFromJsonlAsync_ParsesMetadata() + { + var jsonl = """{"id":"rule1","pattern":"test","metadata":{"source":"gitleaks","category":"api-key"}}"""; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonl)); + + var ruleset = await _loader.LoadFromJsonlAsync( + stream, + "test", + "1.0", + TestContext.Current.CancellationToken); + + ruleset.Rules[0].Metadata.Should().Contain("source", "gitleaks"); + ruleset.Rules[0].Metadata.Should().Contain("category", "api-key"); + } + + private async Task CreateValidBundleAsync(CancellationToken ct) + { + var rules = """ + {"id":"aws-key","version":"1.0","name":"AWS Access Key","type":"regex","pattern":"AKIA[0-9A-Z]{16}","severity":"critical","confidence":"high"} + {"id":"generic-secret","version":"1.0","name":"Generic Secret","type":"regex","pattern":"secret[_-]?key","severity":"medium","confidence":"medium"} + """; + + var rulesPath = Path.Combine(_testDir, "secrets.ruleset.rules.jsonl"); + await File.WriteAllTextAsync(rulesPath, rules, ct); + + var hash = await ComputeHashAsync(rulesPath, ct); + var manifest = $$$"""{"id":"test-secrets","version":"1.0.0","integrity":{"rulesSha256":"{{{hash}}}"}}"""; + + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.manifest.json"), + manifest, + ct); + } + + private async Task CreateBundleWithBadIntegrityAsync(CancellationToken ct) + { + var rules = """{"id":"rule1","pattern":"test"}"""; + var rulesPath = Path.Combine(_testDir, "secrets.ruleset.rules.jsonl"); + await File.WriteAllTextAsync(rulesPath, rules, ct); + + // Use a known bad hash (clearly different from any real SHA-256) + const string badHash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + var manifest = $$$"""{"id":"test","version":"1.0","integrity":{"rulesSha256":"{{{badHash}}}"}}"""; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.manifest.json"), + manifest, + ct); + } + + private async Task CreateBundleWithUnorderedRulesAsync(CancellationToken ct) + { + var rules = """ + {"id":"z-rule","pattern":"z"} + {"id":"a-rule","pattern":"a"} + {"id":"m-rule","pattern":"m"} + """; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.rules.jsonl"), + rules, + ct); + + var manifest = """{"id":"test","version":"1.0"}"""; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.manifest.json"), + manifest, + ct); + } + + private async Task CreateBundleWithBlankLinesAsync(CancellationToken ct) + { + var rules = """ + {"id":"rule1","pattern":"test1"} + + {"id":"rule2","pattern":"test2"} + + """; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.rules.jsonl"), + rules, + ct); + + var manifest = """{"id":"test","version":"1.0"}"""; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.manifest.json"), + manifest, + ct); + } + + private async Task CreateBundleWithInvalidJsonAsync(CancellationToken ct) + { + var rules = """ + {"id":"rule1","pattern":"valid"} + not valid json at all + {"id":"rule2","pattern":"also valid but will be skipped due to earlier error?"} + """; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.rules.jsonl"), + rules, + ct); + + var manifest = """{"id":"test","version":"1.0"}"""; + await File.WriteAllTextAsync( + Path.Combine(_testDir, "secrets.ruleset.manifest.json"), + manifest, + ct); + } + + private static async Task ComputeHashAsync(string filePath, CancellationToken ct) + { + await using var stream = File.OpenRead(filePath); + var hash = await SHA256.HashDataAsync(stream, ct); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRuleTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRuleTests.cs new file mode 100644 index 000000000..19f64dca7 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRuleTests.cs @@ -0,0 +1,173 @@ +using System.Collections.Immutable; +using FluentAssertions; +using StellaOps.Scanner.Analyzers.Secrets; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests; + +[Trait("Category", "Unit")] +public sealed class SecretRuleTests +{ + [Fact] + public void GetCompiledPattern_ValidRegex_ReturnsRegex() + { + var rule = CreateRule(@"AKIA[0-9A-Z]{16}"); + + var regex = rule.GetCompiledPattern(); + + regex.Should().NotBeNull(); + regex!.IsMatch("AKIAIOSFODNN7EXAMPLE").Should().BeTrue(); + } + + [Fact] + public void GetCompiledPattern_InvalidRegex_ReturnsNull() + { + var rule = CreateRule(@"[invalid(regex"); + + var regex = rule.GetCompiledPattern(); + + regex.Should().BeNull(); + } + + [Fact] + public void GetCompiledPattern_IsCached() + { + var rule = CreateRule(@"test\d+"); + + var regex1 = rule.GetCompiledPattern(); + var regex2 = rule.GetCompiledPattern(); + + regex1.Should().BeSameAs(regex2); + } + + [Fact] + public void GetCompiledPattern_EntropyType_ReturnsNull() + { + var rule = new SecretRule + { + Id = "entropy-rule", + Version = "1.0.0", + Name = "Entropy Rule", + Description = "Test", + Type = SecretRuleType.Entropy, + Pattern = string.Empty, + Severity = SecretSeverity.Medium, + Confidence = SecretConfidence.Medium, + Keywords = ImmutableArray.Empty, + FilePatterns = ImmutableArray.Empty, + Enabled = true, + EntropyThreshold = 4.5, + MinLength = 16, + MaxLength = 100, + Metadata = ImmutableDictionary.Empty + }; + + var regex = rule.GetCompiledPattern(); + + regex.Should().BeNull(); + } + + [Fact] + public void AppliesToFile_NoPatterns_MatchesAll() + { + var rule = CreateRule(@"test"); + + rule.AppliesToFile("any/path/file.txt").Should().BeTrue(); + rule.AppliesToFile("config.json").Should().BeTrue(); + rule.AppliesToFile("secrets.yaml").Should().BeTrue(); + } + + [Fact] + public void AppliesToFile_WithExtensionPattern_FiltersByExtension() + { + var rule = CreateRuleWithFilePatterns(@"test", "*.json", "*.yaml"); + + rule.AppliesToFile("config.json").Should().BeTrue(); + rule.AppliesToFile("config.yaml").Should().BeTrue(); + rule.AppliesToFile("config.xml").Should().BeFalse(); + rule.AppliesToFile("config.txt").Should().BeFalse(); + } + + [Fact] + public void MightMatch_NoKeywords_ReturnsTrue() + { + var rule = CreateRule(@"test"); + + rule.MightMatch("any content here".AsSpan()).Should().BeTrue(); + } + + [Fact] + public void MightMatch_WithKeywords_MatchesIfKeywordFound() + { + var rule = CreateRuleWithKeywords(@"test", "secret", "password"); + + rule.MightMatch("contains secret here".AsSpan()).Should().BeTrue(); + rule.MightMatch("contains password here".AsSpan()).Should().BeTrue(); + rule.MightMatch("no matching content".AsSpan()).Should().BeFalse(); + } + + private static SecretRule CreateRule(string pattern) + { + return new SecretRule + { + Id = "test-rule", + Version = "1.0.0", + Name = "Test Rule", + Description = "Test rule", + Type = SecretRuleType.Regex, + Pattern = pattern, + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = ImmutableArray.Empty, + FilePatterns = ImmutableArray.Empty, + Enabled = true, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + } + + private static SecretRule CreateRuleWithFilePatterns(string pattern, params string[] filePatterns) + { + return new SecretRule + { + Id = "test-rule", + Version = "1.0.0", + Name = "Test Rule", + Description = "Test rule", + Type = SecretRuleType.Regex, + Pattern = pattern, + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = ImmutableArray.Empty, + FilePatterns = [..filePatterns], + Enabled = true, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + } + + private static SecretRule CreateRuleWithKeywords(string pattern, params string[] keywords) + { + return new SecretRule + { + Id = "test-rule", + Version = "1.0.0", + Name = "Test Rule", + Description = "Test rule", + Type = SecretRuleType.Regex, + Pattern = pattern, + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = [..keywords], + FilePatterns = ImmutableArray.Empty, + Enabled = true, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRulesetTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRulesetTests.cs new file mode 100644 index 000000000..94ae2335c --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/SecretRulesetTests.cs @@ -0,0 +1,208 @@ +using System.Collections.Immutable; +using FluentAssertions; +using StellaOps.Scanner.Analyzers.Secrets; + +namespace StellaOps.Scanner.Analyzers.Secrets.Tests; + +[Trait("Category", "Unit")] +public sealed class SecretRulesetTests +{ + [Fact] + public void EnabledRuleCount_ReturnsCorrectCount() + { + var ruleset = CreateRuleset( + CreateRule("rule1", enabled: true), + CreateRule("rule2", enabled: true), + CreateRule("rule3", enabled: false)); + + ruleset.EnabledRuleCount.Should().Be(2); + } + + [Fact] + public void EnabledRuleCount_AllDisabled_ReturnsZero() + { + var ruleset = CreateRuleset( + CreateRule("rule1", enabled: false), + CreateRule("rule2", enabled: false)); + + ruleset.EnabledRuleCount.Should().Be(0); + } + + [Fact] + public void EnabledRuleCount_AllEnabled_ReturnsTotal() + { + var ruleset = CreateRuleset( + CreateRule("rule1", enabled: true), + CreateRule("rule2", enabled: true), + CreateRule("rule3", enabled: true)); + + ruleset.EnabledRuleCount.Should().Be(3); + } + + [Fact] + public void GetRulesForFile_ReturnsEnabledMatchingRules() + { + var ruleset = CreateRuleset( + CreateRuleWithPattern("json-rule", "*.json", enabled: true), + CreateRuleWithPattern("yaml-rule", "*.yaml", enabled: true), + CreateRuleWithPattern("disabled-rule", "*.json", enabled: false)); + + var rules = ruleset.GetRulesForFile("config.json").ToList(); + + rules.Should().HaveCount(1); + rules[0].Id.Should().Be("json-rule"); + } + + [Fact] + public void GetRulesForFile_NoMatchingPatterns_ReturnsRulesWithNoPatterns() + { + var ruleset = CreateRuleset( + CreateRule("generic-rule", enabled: true), + CreateRuleWithPattern("json-rule", "*.json", enabled: true)); + + var rules = ruleset.GetRulesForFile("config.xml").ToList(); + + rules.Should().HaveCount(1); + rules[0].Id.Should().Be("generic-rule"); + } + + [Fact] + public void Validate_ValidRuleset_ReturnsEmpty() + { + var ruleset = CreateRuleset( + CreateRule("rule1", enabled: true), + CreateRule("rule2", enabled: true)); + + var errors = ruleset.Validate(); + + errors.Should().BeEmpty(); + } + + [Fact] + public void Validate_DuplicateIds_ReturnsError() + { + var ruleset = CreateRuleset( + CreateRule("same-id", enabled: true), + CreateRule("same-id", enabled: true)); + + var errors = ruleset.Validate(); + + errors.Should().Contain(e => e.Contains("Duplicate", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_InvalidRegex_ReturnsError() + { + var rule = new SecretRule + { + Id = "bad-regex", + Version = "1.0.0", + Name = "Bad Regex", + Description = "Invalid regex pattern", + Type = SecretRuleType.Regex, + Pattern = "[invalid(regex", + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = ImmutableArray.Empty, + FilePatterns = ImmutableArray.Empty, + Enabled = true, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + + var ruleset = new SecretRuleset + { + Id = "test", + Version = "1.0.0", + CreatedAt = DateTimeOffset.UtcNow, + Rules = [rule] + }; + + var errors = ruleset.Validate(); + + errors.Should().Contain(e => e.Contains("bad-regex", StringComparison.OrdinalIgnoreCase) && + e.Contains("invalid", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void EnabledRules_ReturnsOnlyEnabled() + { + var ruleset = CreateRuleset( + CreateRule("rule1", enabled: true), + CreateRule("rule2", enabled: false), + CreateRule("rule3", enabled: true)); + + var enabled = ruleset.EnabledRules.ToList(); + + enabled.Should().HaveCount(2); + enabled.Select(r => r.Id).Should().BeEquivalentTo(["rule1", "rule3"]); + } + + [Fact] + public void Empty_ReturnsEmptyRuleset() + { + var empty = SecretRuleset.Empty; + + empty.Id.Should().Be("empty"); + empty.Version.Should().Be("0.0"); + empty.Rules.Should().BeEmpty(); + empty.EnabledRuleCount.Should().Be(0); + } + + private static SecretRuleset CreateRuleset(params SecretRule[] rules) + { + return new SecretRuleset + { + Id = "test-ruleset", + Version = "1.0.0", + CreatedAt = DateTimeOffset.UtcNow, + Rules = [..rules] + }; + } + + private static SecretRule CreateRule(string id, bool enabled) + { + return new SecretRule + { + Id = id, + Version = "1.0.0", + Name = $"Rule {id}", + Description = "Test rule", + Type = SecretRuleType.Regex, + Pattern = "test", + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = ImmutableArray.Empty, + FilePatterns = ImmutableArray.Empty, + Enabled = enabled, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + } + + private static SecretRule CreateRuleWithPattern(string id, string filePattern, bool enabled) + { + return new SecretRule + { + Id = id, + Version = "1.0.0", + Name = $"Rule {id}", + Description = "Test rule", + Type = SecretRuleType.Regex, + Pattern = "test", + Severity = SecretSeverity.High, + Confidence = SecretConfidence.High, + Keywords = ImmutableArray.Empty, + FilePatterns = [filePattern], + Enabled = enabled, + EntropyThreshold = 0, + MinLength = 0, + MaxLength = 1000, + Metadata = ImmutableDictionary.Empty + }; + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/StellaOps.Scanner.Analyzers.Secrets.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/StellaOps.Scanner.Analyzers.Secrets.Tests.csproj new file mode 100644 index 000000000..785e58d16 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Secrets.Tests/StellaOps.Scanner.Analyzers.Secrets.Tests.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + preview + enable + enable + true + false + true + + + + + + + + + + + + + + + + + + + PreserveNewest + + + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.CallGraph.Tests/CallGraphDigestsTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.CallGraph.Tests/CallGraphDigestsTests.cs new file mode 100644 index 000000000..9e76d3bc9 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.CallGraph.Tests/CallGraphDigestsTests.cs @@ -0,0 +1,469 @@ +// ----------------------------------------------------------------------------- +// CallGraphDigestsTests.cs +// Sprint: SPRINT_20260104_001_CLI +// Description: Unit tests for call graph digest computation and determinism. +// ----------------------------------------------------------------------------- + +using System.Collections.Immutable; +using StellaOps.Scanner.CallGraph; +using StellaOps.Scanner.Reachability; +using StellaOps.TestKit; +using Xunit; + +namespace StellaOps.Scanner.CallGraph.Tests; + +[Trait("Category", TestCategories.Unit)] +public class CallGraphDigestsTests +{ + [Fact] + public void ComputeGraphDigest_ReturnsValidSha256Format() + { + // Arrange + var snapshot = CreateMinimalSnapshot(); + + // Act + var digest = CallGraphDigests.ComputeGraphDigest(snapshot); + + // Assert + Assert.NotNull(digest); + Assert.StartsWith("sha256:", digest, StringComparison.Ordinal); + Assert.Equal(71, digest.Length); // "sha256:" (7) + 64 hex chars + Assert.True(IsValidHex(digest[7..]), "Digest should be valid hex string"); + } + + [Fact] + public void ComputeGraphDigest_IsDeterministic() + { + // Arrange + var snapshot = CreateMinimalSnapshot(); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot); + var digest3 = CallGraphDigests.ComputeGraphDigest(snapshot); + + // Assert + Assert.Equal(digest1, digest2); + Assert.Equal(digest2, digest3); + } + + [Fact] + public void ComputeGraphDigest_EquivalentSnapshotsProduceSameDigest() + { + // Arrange - two separately created but equivalent snapshots + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan-1", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, true, EntrypointType.CliCommand, false, null), + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Internal, false, null, false, null) + ), + Edges: ImmutableArray.Create( + new CallGraphEdge("node-a", "node-b", CallKind.Direct) + ), + EntrypointIds: ImmutableArray.Create("node-a"), + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: "test-scan-1", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow.AddMinutes(5), // Different timestamp + Nodes: ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, true, EntrypointType.CliCommand, false, null), + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Internal, false, null, false, null) + ), + Edges: ImmutableArray.Create( + new CallGraphEdge("node-a", "node-b", CallKind.Direct) + ), + EntrypointIds: ImmutableArray.Create("node-a"), + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert - digests should match because ExtractedAt is not part of the digest payload + Assert.Equal(digest1, digest2); + } + + [Fact] + public void ComputeGraphDigest_DifferentNodesProduceDifferentDigests() + { + // Arrange + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, false, null, false, null) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Public, false, null, false, null) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert + Assert.NotEqual(digest1, digest2); + } + + [Fact] + public void ComputeGraphDigest_NodeOrderDoesNotAffectDigest() + { + // Arrange - nodes in different order + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, false, null, false, null), + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Public, false, null, false, null) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Public, false, null, false, null), + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, false, null, false, null) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert - digests should match because Trimmed() sorts nodes + Assert.Equal(digest1, digest2); + } + + [Fact] + public void ComputeGraphDigest_EdgeOrderDoesNotAffectDigest() + { + // Arrange - edges in different order + var nodes = ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, true, EntrypointType.CliCommand, false, null), + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Internal, false, null, false, null), + new CallGraphNode("node-c", "func_c", "test.c", 30, "pkg", Visibility.Internal, false, null, false, null) + ); + + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: nodes, + Edges: ImmutableArray.Create( + new CallGraphEdge("node-a", "node-b", CallKind.Direct), + new CallGraphEdge("node-a", "node-c", CallKind.Direct) + ), + EntrypointIds: ImmutableArray.Create("node-a"), + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: nodes, + Edges: ImmutableArray.Create( + new CallGraphEdge("node-a", "node-c", CallKind.Direct), + new CallGraphEdge("node-a", "node-b", CallKind.Direct) + ), + EntrypointIds: ImmutableArray.Create("node-a"), + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert - digests should match because Trimmed() sorts edges + Assert.Equal(digest1, digest2); + } + + [Fact] + public void ComputeGraphDigest_WhitespaceIsTrimmed() + { + // Arrange + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, false, null, false, null) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: " test-scan ", + GraphDigest: "", + Language: " native ", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode(" node-a ", " func_a ", " test.c ", 10, " pkg ", Visibility.Public, false, null, false, null) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert - digests should match because Trimmed() trims whitespace + Assert.Equal(digest1, digest2); + } + + [Fact] + public void ComputeGraphDigest_ThrowsOnNull() + { + // Act & Assert + Assert.Throws(() => CallGraphDigests.ComputeGraphDigest(null!)); + } + + [Fact] + public void ComputeGraphDigest_HandlesEmptySnapshot() + { + // Arrange + var snapshot = new CallGraphSnapshot( + ScanId: "", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Empty, + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest = CallGraphDigests.ComputeGraphDigest(snapshot); + + // Assert + Assert.NotNull(digest); + Assert.StartsWith("sha256:", digest, StringComparison.Ordinal); + } + + [Fact] + public void ComputeGraphDigest_LanguageAffectsDigest() + { + // Arrange + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Empty, + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "dotnet", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Empty, + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert + Assert.NotEqual(digest1, digest2); + } + + [Fact] + public void ComputeGraphDigest_EdgeExplanationAffectsDigest() + { + // Arrange + var nodes = ImmutableArray.Create( + new CallGraphNode("node-a", "func_a", "test.c", 10, "pkg", Visibility.Public, false, null, false, null), + new CallGraphNode("node-b", "func_b", "test.c", 20, "pkg", Visibility.Public, false, null, false, null) + ); + + var snapshot1 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: nodes, + Edges: ImmutableArray.Create( + new CallGraphEdge("node-a", "node-b", CallKind.Direct, null, null) + ), + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + var snapshot2 = new CallGraphSnapshot( + ScanId: "test-scan", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: nodes, + Edges: ImmutableArray.Create( + new CallGraphEdge("node-a", "node-b", CallKind.Direct, null, CallEdgeExplanation.DirectCall()) + ), + EntrypointIds: ImmutableArray.Empty, + SinkIds: ImmutableArray.Empty + ); + + // Act + var digest1 = CallGraphDigests.ComputeGraphDigest(snapshot1); + var digest2 = CallGraphDigests.ComputeGraphDigest(snapshot2); + + // Assert + Assert.NotEqual(digest1, digest2); + } + + [Fact] + public void CallGraphNodeIds_Compute_ReturnsValidSha256Format() + { + // Arrange + var stableId = "native:main"; + + // Act + var nodeId = CallGraphNodeIds.Compute(stableId); + + // Assert + Assert.NotNull(nodeId); + Assert.StartsWith("sha256:", nodeId, StringComparison.Ordinal); + Assert.Equal(71, nodeId.Length); + } + + [Fact] + public void CallGraphNodeIds_Compute_IsDeterministic() + { + // Arrange + var stableId = "native:SSL_read"; + + // Act + var id1 = CallGraphNodeIds.Compute(stableId); + var id2 = CallGraphNodeIds.Compute(stableId); + var id3 = CallGraphNodeIds.Compute(stableId); + + // Assert + Assert.Equal(id1, id2); + Assert.Equal(id2, id3); + } + + [Fact] + public void CallGraphNodeIds_Compute_DifferentSymbolsProduceDifferentIds() + { + // Arrange + var stableId1 = "native:func_a"; + var stableId2 = "native:func_b"; + + // Act + var id1 = CallGraphNodeIds.Compute(stableId1); + var id2 = CallGraphNodeIds.Compute(stableId2); + + // Assert + Assert.NotEqual(id1, id2); + } + + [Fact] + public void CallGraphNodeIds_StableSymbolId_CreatesConsistentFormat() + { + // Arrange & Act + var stableId = CallGraphNodeIds.StableSymbolId("Native", "SSL_read"); + + // Assert + Assert.Equal("native:SSL_read", stableId); + } + + [Fact] + public void CallGraphNodeIds_StableSymbolId_TrimsWhitespace() + { + // Arrange & Act + var stableId = CallGraphNodeIds.StableSymbolId(" Native ", " SSL_read "); + + // Assert + Assert.Equal("native:SSL_read", stableId); + } + + private static CallGraphSnapshot CreateMinimalSnapshot() + { + return new CallGraphSnapshot( + ScanId: "test-scan-001", + GraphDigest: "", + Language: "native", + ExtractedAt: DateTimeOffset.UtcNow, + Nodes: ImmutableArray.Create( + new CallGraphNode( + NodeId: "sha256:abc123", + Symbol: "main", + File: "main.c", + Line: 1, + Package: "test-binary", + Visibility: Visibility.Public, + IsEntrypoint: true, + EntrypointType: EntrypointType.CliCommand, + IsSink: false, + SinkCategory: null + ) + ), + Edges: ImmutableArray.Empty, + EntrypointIds: ImmutableArray.Create("sha256:abc123"), + SinkIds: ImmutableArray.Empty + ); + } + + private static bool IsValidHex(string hex) + { + if (string.IsNullOrEmpty(hex)) + return false; + + foreach (char c in hex) + { + if (!char.IsAsciiHexDigit(c)) + return false; + } + + return true; + } +} diff --git a/src/VexLens/StellaOps.VexLens.WebService/Extensions/VexLensEndpointExtensions.cs b/src/VexLens/StellaOps.VexLens.WebService/Extensions/VexLensEndpointExtensions.cs index abeb210b9..8523a50cf 100644 --- a/src/VexLens/StellaOps.VexLens.WebService/Extensions/VexLensEndpointExtensions.cs +++ b/src/VexLens/StellaOps.VexLens.WebService/Extensions/VexLensEndpointExtensions.cs @@ -1,5 +1,8 @@ using Microsoft.AspNetCore.Mvc; using StellaOps.VexLens.Api; +using StellaOps.VexLens.Delta; +using StellaOps.VexLens.NoiseGate; +using StellaOps.VexLens.Storage; namespace StellaOps.VexLens.WebService.Extensions; @@ -73,6 +76,32 @@ public static class VexLensEndpointExtensions .WithDescription("Get projections with conflicts") .Produces(StatusCodes.Status200OK); + // Delta/Noise-Gating endpoints + var deltaGroup = app.MapGroup("/api/v1/vexlens/deltas") + .WithTags("VexLens Delta") + .WithOpenApi(); + + deltaGroup.MapPost("/compute", ComputeDeltaAsync) + .WithName("ComputeDelta") + .WithDescription("Compute delta report between two snapshots") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest); + + var gatingGroup = app.MapGroup("/api/v1/vexlens/gating") + .WithTags("VexLens Gating") + .WithOpenApi(); + + gatingGroup.MapGet("/statistics", GetGatingStatisticsAsync) + .WithName("GetGatingStatistics") + .WithDescription("Get aggregated noise-gating statistics") + .Produces(StatusCodes.Status200OK); + + gatingGroup.MapPost("/snapshots/{snapshotId}/gate", GateSnapshotAsync) + .WithName("GateSnapshot") + .WithDescription("Apply noise-gating to a snapshot") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound); + // Issuer endpoints var issuerGroup = app.MapGroup("/api/v1/vexlens/issuers") .WithTags("VexLens Issuers") @@ -265,6 +294,91 @@ public static class VexLensEndpointExtensions return Results.Ok(conflictsOnly); } + // Delta/Noise-Gating handlers + private static async Task ComputeDeltaAsync( + [FromBody] ComputeDeltaRequest request, + [FromServices] INoiseGate noiseGate, + [FromServices] ISnapshotStore snapshotStore, + HttpContext context, + CancellationToken cancellationToken) + { + var tenantId = GetTenantId(context) ?? request.TenantId; + + // Get snapshots + var fromSnapshot = await snapshotStore.GetAsync(request.FromSnapshotId, tenantId, cancellationToken); + var toSnapshot = await snapshotStore.GetAsync(request.ToSnapshotId, tenantId, cancellationToken); + + if (fromSnapshot is null || toSnapshot is null) + { + return Results.BadRequest("One or both snapshot IDs not found"); + } + + // Compute delta + var options = NoiseGatingApiMapper.MapOptions(request.Options); + var delta = await noiseGate.DiffAsync(fromSnapshot, toSnapshot, options, cancellationToken); + + return Results.Ok(NoiseGatingApiMapper.MapToResponse(delta)); + } + + private static async Task GetGatingStatisticsAsync( + [FromQuery] string? tenantId, + [FromQuery] DateTimeOffset? fromDate, + [FromQuery] DateTimeOffset? toDate, + [FromServices] IGatingStatisticsStore statsStore, + HttpContext context, + CancellationToken cancellationToken) + { + var tenant = GetTenantId(context) ?? tenantId; + var stats = await statsStore.GetAggregatedAsync(tenant, fromDate, toDate, cancellationToken); + + return Results.Ok(new AggregatedGatingStatisticsResponse( + TotalSnapshots: stats.TotalSnapshots, + TotalEdgesProcessed: stats.TotalEdgesProcessed, + TotalEdgesAfterDedup: stats.TotalEdgesAfterDedup, + AverageEdgeReductionPercent: stats.AverageEdgeReductionPercent, + TotalVerdicts: stats.TotalVerdicts, + TotalSurfaced: stats.TotalSurfaced, + TotalDamped: stats.TotalDamped, + AverageDampingPercent: stats.AverageDampingPercent, + ComputedAt: DateTimeOffset.UtcNow)); + } + + private static async Task GateSnapshotAsync( + string snapshotId, + [FromBody] GateSnapshotRequest request, + [FromServices] INoiseGate noiseGate, + [FromServices] ISnapshotStore snapshotStore, + HttpContext context, + CancellationToken cancellationToken) + { + var tenantId = GetTenantId(context) ?? request.TenantId; + + // Get the raw snapshot + var snapshot = await snapshotStore.GetRawAsync(snapshotId, tenantId, cancellationToken); + if (snapshot is null) + { + return Results.NotFound(); + } + + // Apply noise-gating + var gateRequest = new NoiseGateRequest + { + Graph = snapshot.Graph, + SnapshotId = snapshotId, + Verdicts = snapshot.Verdicts + }; + + var gatedSnapshot = await noiseGate.GateAsync(gateRequest, cancellationToken); + + return Results.Ok(new GatedSnapshotResponse( + SnapshotId: gatedSnapshot.SnapshotId, + Digest: gatedSnapshot.Digest, + CreatedAt: gatedSnapshot.CreatedAt, + EdgeCount: gatedSnapshot.Edges.Count, + VerdictCount: gatedSnapshot.Verdicts.Count, + Statistics: NoiseGatingApiMapper.MapStatistics(gatedSnapshot.Statistics))); + } + // Issuer handlers private static async Task ListIssuersAsync( [FromQuery] string? category, diff --git a/src/VexLens/StellaOps.VexLens/AGENTS.md b/src/VexLens/StellaOps.VexLens/AGENTS.md index 499a7f0ce..6a6afd854 100644 --- a/src/VexLens/StellaOps.VexLens/AGENTS.md +++ b/src/VexLens/StellaOps.VexLens/AGENTS.md @@ -7,6 +7,12 @@ Deliver the VEX Consensus Lens service that normalizes VEX evidence, computes de - Service code under `src/VexLens/StellaOps.VexLens` (normalizer, mapping, trust weighting, consensus projection, APIs, simulation hooks). - Batch workers consuming Excitor, Conseiller, SBOM, and policy events; projection storage and caching; telemetry. - Coordination with Policy Engine, Vuln Explorer, Findings Ledger, Console, CLI, and Docs. +- **NoiseGate** (Sprint NG-001): Unified noise-gating for vulnerability graphs: + - **INoiseGate**: Central interface for noise-gating operations + - **EdgeDeduplicator**: Collapses semantically equivalent edges (uses StellaOps.ReachGraph) + - **StabilityDampingGate**: Hysteresis-based damping (uses StellaOps.Policy.Engine.Gates) + - **DeltaReport**: Typed sections (New, Resolved, ConfidenceUp/Down, PolicyImpact, Damped, EvidenceChanged) + - **DeltaReportBuilder**: Fluent builder for change reports with deterministic output ## Principles 1. **Evidence preserving** – never edit or merge raw VEX docs; link via evidence IDs and maintain provenance. diff --git a/src/VexLens/StellaOps.VexLens/Api/NoiseGatingApiModels.cs b/src/VexLens/StellaOps.VexLens/Api/NoiseGatingApiModels.cs new file mode 100644 index 000000000..349d103a0 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Api/NoiseGatingApiModels.cs @@ -0,0 +1,205 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using StellaOps.VexLens.Delta; +using StellaOps.VexLens.Models; +using StellaOps.VexLens.NoiseGate; + +namespace StellaOps.VexLens.Api; + +/// +/// Request to compute delta between two snapshots. +/// +public sealed record ComputeDeltaRequest( + string FromSnapshotId, + string ToSnapshotId, + string? TenantId, + DeltaReportOptionsRequest? Options); + +/// +/// Options for delta report computation. +/// +public sealed record DeltaReportOptionsRequest( + double? ConfidenceChangeThreshold, + bool? IncludeDamped, + bool? IncludeEvidenceChanges); + +/// +/// Response from delta computation. +/// +public sealed record DeltaReportResponse( + string ReportId, + string FromSnapshotDigest, + string ToSnapshotDigest, + DateTimeOffset GeneratedAt, + IReadOnlyList Entries, + DeltaSummaryResponse Summary, + bool HasActionableChanges); + +/// +/// Summary counts for delta report. +/// +public sealed record DeltaSummaryResponse( + int TotalCount, + int NewCount, + int ResolvedCount, + int ConfidenceUpCount, + int ConfidenceDownCount, + int PolicyImpactCount, + int DampedCount, + int EvidenceChangedCount); + +/// +/// Single delta entry in API format. +/// +public sealed record DeltaEntryResponse( + string Section, + string VulnerabilityId, + string ProductKey, + string? FromStatus, + string? ToStatus, + double? FromConfidence, + double? ToConfidence, + string? Justification, + string? FromRationaleClass, + string? ToRationaleClass, + string? Summary, + IReadOnlyList? ContributingSources, + DateTimeOffset CreatedAt); + +/// +/// Request to gate a graph snapshot. +/// +public sealed record GateSnapshotRequest( + string SnapshotId, + string? TenantId, + NoiseGateOptionsRequest? Options); + +/// +/// Options for noise-gating. +/// +public sealed record NoiseGateOptionsRequest( + bool? EdgeDeduplicationEnabled, + bool? StabilityDampingEnabled, + double? MinConfidenceThreshold, + double? ConfidenceChangeThreshold); + +/// +/// Response from gating a snapshot. +/// +public sealed record GatedSnapshotResponse( + string SnapshotId, + string Digest, + DateTimeOffset CreatedAt, + int EdgeCount, + int VerdictCount, + GatingStatisticsResponse Statistics); + +/// +/// Gating statistics for API response. +/// +public sealed record GatingStatisticsResponse( + int OriginalEdgeCount, + int DeduplicatedEdgeCount, + double EdgeReductionPercent, + int TotalVerdictCount, + int SurfacedVerdictCount, + int DampedVerdictCount, + string Duration); + +/// +/// Request to get aggregated gating statistics. +/// +public sealed record GatingStatisticsQueryRequest( + string? TenantId, + DateTimeOffset? FromDate, + DateTimeOffset? ToDate); + +/// +/// Aggregated gating statistics response. +/// +public sealed record AggregatedGatingStatisticsResponse( + int TotalSnapshots, + int TotalEdgesProcessed, + int TotalEdgesAfterDedup, + double AverageEdgeReductionPercent, + int TotalVerdicts, + int TotalSurfaced, + int TotalDamped, + double AverageDampingPercent, + DateTimeOffset ComputedAt); + +/// +/// Maps internal delta models to API responses. +/// +internal static class NoiseGatingApiMapper +{ + public static DeltaReportResponse MapToResponse(DeltaReport report) + { + return new DeltaReportResponse( + ReportId: report.ReportId, + FromSnapshotDigest: report.FromSnapshotDigest, + ToSnapshotDigest: report.ToSnapshotDigest, + GeneratedAt: report.GeneratedAt, + Entries: report.Entries.Select(MapEntry).ToList(), + Summary: MapSummary(report.Summary), + HasActionableChanges: report.HasActionableChanges); + } + + public static DeltaSummaryResponse MapSummary(DeltaSummary summary) + { + return new DeltaSummaryResponse( + TotalCount: summary.TotalCount, + NewCount: summary.NewCount, + ResolvedCount: summary.ResolvedCount, + ConfidenceUpCount: summary.ConfidenceUpCount, + ConfidenceDownCount: summary.ConfidenceDownCount, + PolicyImpactCount: summary.PolicyImpactCount, + DampedCount: summary.DampedCount, + EvidenceChangedCount: summary.EvidenceChangedCount); + } + + public static DeltaEntryResponse MapEntry(DeltaEntry entry) + { + return new DeltaEntryResponse( + Section: entry.Section.ToString().ToLowerInvariant(), + VulnerabilityId: entry.VulnerabilityId, + ProductKey: entry.ProductKey, + FromStatus: entry.FromStatus?.ToString(), + ToStatus: entry.ToStatus?.ToString(), + FromConfidence: entry.FromConfidence, + ToConfidence: entry.ToConfidence, + Justification: entry.Justification?.ToString(), + FromRationaleClass: entry.FromRationaleClass, + ToRationaleClass: entry.ToRationaleClass, + Summary: entry.Summary, + ContributingSources: entry.ContributingSources?.ToList(), + CreatedAt: entry.Timestamp); + } + + public static GatingStatisticsResponse MapStatistics(GatingStatistics stats) + { + return new GatingStatisticsResponse( + OriginalEdgeCount: stats.OriginalEdgeCount, + DeduplicatedEdgeCount: stats.DeduplicatedEdgeCount, + EdgeReductionPercent: stats.EdgeReductionPercent, + TotalVerdictCount: stats.TotalVerdictCount, + SurfacedVerdictCount: stats.SurfacedVerdictCount, + DampedVerdictCount: stats.DampedVerdictCount, + Duration: stats.Duration.ToString("c")); + } + + public static DeltaReportOptions MapOptions(DeltaReportOptionsRequest? request) + { + if (request is null) + { + return new DeltaReportOptions(); + } + + return new DeltaReportOptions + { + ConfidenceChangeThreshold = request.ConfidenceChangeThreshold ?? 0.15, + IncludeDamped = request.IncludeDamped ?? true, + IncludeEvidenceChanges = request.IncludeEvidenceChanges ?? true + }; + } +} diff --git a/src/VexLens/StellaOps.VexLens/Delta/DeltaEntry.cs b/src/VexLens/StellaOps.VexLens/Delta/DeltaEntry.cs new file mode 100644 index 000000000..f94fc1797 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Delta/DeltaEntry.cs @@ -0,0 +1,88 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using StellaOps.VexLens.Models; + +namespace StellaOps.VexLens.Delta; + +/// +/// A single entry in a delta report representing a change between snapshots. +/// +public sealed record DeltaEntry +{ + /// + /// Gets the unique identifier for this delta entry. + /// + public required string DeltaId { get; init; } + + /// + /// Gets the vulnerability identifier. + /// + public required string VulnerabilityId { get; init; } + + /// + /// Gets the product/component key (typically PURL). + /// + public required string ProductKey { get; init; } + + /// + /// Gets the section this delta belongs to. + /// + public required DeltaSection Section { get; init; } + + /// + /// Gets the previous VEX status, if any. + /// + public VexStatus? FromStatus { get; init; } + + /// + /// Gets the current VEX status. + /// + public required VexStatus ToStatus { get; init; } + + /// + /// Gets the previous confidence score, if any. + /// + public double? FromConfidence { get; init; } + + /// + /// Gets the current confidence score. + /// + public required double ToConfidence { get; init; } + + /// + /// Gets the previous rationale class, if any. + /// + public string? FromRationaleClass { get; init; } + + /// + /// Gets the current rationale class. + /// + public string? ToRationaleClass { get; init; } + + /// + /// Gets the justification for the current status. + /// + public VexJustification? Justification { get; init; } + + /// + /// Gets a human-readable summary of the change. + /// + public required string Summary { get; init; } + + /// + /// Gets the timestamp of this delta. + /// + public required DateTimeOffset Timestamp { get; init; } + + /// + /// Gets the sources that contributed to this change. + /// + public ImmutableArray ContributingSources { get; init; } = []; + + /// + /// Gets additional metadata. + /// + public ImmutableDictionary Metadata { get; init; } = + ImmutableDictionary.Empty; +} diff --git a/src/VexLens/StellaOps.VexLens/Delta/DeltaReport.cs b/src/VexLens/StellaOps.VexLens/Delta/DeltaReport.cs new file mode 100644 index 000000000..fe96fbc43 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Delta/DeltaReport.cs @@ -0,0 +1,183 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using System.Globalization; + +namespace StellaOps.VexLens.Delta; + +/// +/// A report summarizing changes between two vulnerability graph snapshots. +/// +/// +/// DeltaReport groups changes by section for efficient triage: +/// - Users can focus on New findings first +/// - Resolved items can be quickly acknowledged +/// - Confidence changes help reprioritize existing findings +/// - Policy impacts highlight workflow-affecting changes +/// +public sealed record DeltaReport +{ + /// + /// Gets the unique identifier for this report. + /// + public required string ReportId { get; init; } + + /// + /// Gets the digest of the previous snapshot. + /// + public required string FromSnapshotDigest { get; init; } + + /// + /// Gets the digest of the current snapshot. + /// + public required string ToSnapshotDigest { get; init; } + + /// + /// Gets the timestamp when the report was generated. + /// + public required DateTimeOffset GeneratedAt { get; init; } + + /// + /// Gets all delta entries in this report. + /// + public required ImmutableArray Entries { get; init; } + + /// + /// Gets the summary counts by section. + /// + public required DeltaSummary Summary { get; init; } + + /// + /// Gets entries grouped by section for UI consumption. + /// + public ImmutableDictionary> BySection => + Entries + .GroupBy(e => e.Section) + .ToImmutableDictionary( + g => g.Key, + g => g.ToImmutableArray()); + + /// + /// Gets entries for a specific section. + /// + public ImmutableArray GetSection(DeltaSection section) => + BySection.TryGetValue(section, out var entries) + ? entries + : []; + + /// + /// Gets whether this report has any actionable changes. + /// + public bool HasActionableChanges => + Summary.NewCount > 0 || + Summary.PolicyImpactCount > 0 || + Summary.ConfidenceUpCount > 0; + + /// + /// Gets a one-line summary suitable for notifications. + /// + public string ToNotificationSummary() + { + var parts = new List(); + + if (Summary.NewCount > 0) + { + parts.Add(string.Create(CultureInfo.InvariantCulture, + $"{Summary.NewCount} new")); + } + + if (Summary.ResolvedCount > 0) + { + parts.Add(string.Create(CultureInfo.InvariantCulture, + $"{Summary.ResolvedCount} resolved")); + } + + if (Summary.PolicyImpactCount > 0) + { + parts.Add(string.Create(CultureInfo.InvariantCulture, + $"{Summary.PolicyImpactCount} policy impact")); + } + + if (Summary.ConfidenceUpCount > 0) + { + parts.Add(string.Create(CultureInfo.InvariantCulture, + $"{Summary.ConfidenceUpCount} confidence up")); + } + + if (Summary.ConfidenceDownCount > 0) + { + parts.Add(string.Create(CultureInfo.InvariantCulture, + $"{Summary.ConfidenceDownCount} confidence down")); + } + + return parts.Count == 0 + ? "No significant changes" + : string.Join(", ", parts); + } +} + +/// +/// Summary counts for a delta report. +/// +public sealed record DeltaSummary +{ + /// + /// Gets the total number of entries. + /// + public required int TotalCount { get; init; } + + /// + /// Gets the count of new findings. + /// + public required int NewCount { get; init; } + + /// + /// Gets the count of resolved findings. + /// + public required int ResolvedCount { get; init; } + + /// + /// Gets the count of confidence increases. + /// + public required int ConfidenceUpCount { get; init; } + + /// + /// Gets the count of confidence decreases. + /// + public required int ConfidenceDownCount { get; init; } + + /// + /// Gets the count of policy impact changes. + /// + public required int PolicyImpactCount { get; init; } + + /// + /// Gets the count of damped (suppressed) changes. + /// + public int DampedCount { get; init; } + + /// + /// Gets the count of evidence-only changes. + /// + public int EvidenceChangedCount { get; init; } + + /// + /// Creates a summary from a list of entries. + /// + public static DeltaSummary FromEntries(IEnumerable entries) + { + var list = entries.ToList(); + + return new DeltaSummary + { + TotalCount = list.Count, + NewCount = list.Count(e => e.Section == DeltaSection.New), + ResolvedCount = list.Count(e => e.Section == DeltaSection.Resolved), + ConfidenceUpCount = list.Count(e => e.Section == DeltaSection.ConfidenceUp), + ConfidenceDownCount = list.Count(e => e.Section == DeltaSection.ConfidenceDown), + PolicyImpactCount = list.Count(e => e.Section == DeltaSection.PolicyImpact), + DampedCount = list.Count(e => e.Section == DeltaSection.Damped), + EvidenceChangedCount = list.Count(e => e.Section == DeltaSection.EvidenceChanged) + }; + } +} diff --git a/src/VexLens/StellaOps.VexLens/Delta/DeltaReportBuilder.cs b/src/VexLens/StellaOps.VexLens/Delta/DeltaReportBuilder.cs new file mode 100644 index 000000000..1a317e836 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Delta/DeltaReportBuilder.cs @@ -0,0 +1,347 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using StellaOps.VexLens.Models; + +namespace StellaOps.VexLens.Delta; + +/// +/// Options for delta report generation. +/// +public sealed record DeltaReportOptions +{ + /// + /// Gets or sets the confidence change threshold for triggering ConfidenceUp/Down sections. + /// + public double ConfidenceChangeThreshold { get; init; } = 0.15; + + /// + /// Gets or sets whether to include damped entries in the report. + /// + public bool IncludeDamped { get; init; } = false; + + /// + /// Gets or sets whether to include evidence-only changes. + /// + public bool IncludeEvidenceChanges { get; init; } = true; +} + +/// +/// Builder for creating instances. +/// +public sealed class DeltaReportBuilder +{ + private readonly List _entries = new(); + private readonly TimeProvider _timeProvider; + private string _fromDigest = string.Empty; + private string _toDigest = string.Empty; + private DeltaReportOptions _options = new(); + + /// + /// Initializes a new delta report builder. + /// + public DeltaReportBuilder(TimeProvider? timeProvider = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + /// Sets the snapshot digests. + /// + public DeltaReportBuilder WithSnapshots(string fromDigest, string toDigest) + { + _fromDigest = fromDigest; + _toDigest = toDigest; + return this; + } + + /// + /// Sets the report options. + /// + public DeltaReportBuilder WithOptions(DeltaReportOptions options) + { + _options = options; + return this; + } + + /// + /// Adds a new finding entry. + /// + public DeltaReportBuilder AddNew( + string vulnerabilityId, + string productKey, + VexStatus status, + double confidence, + string? rationaleClass = null, + VexJustification? justification = null, + IEnumerable? sources = null) + { + var entry = CreateEntry( + vulnerabilityId, + productKey, + DeltaSection.New, + null, + status, + null, + confidence, + null, + rationaleClass, + justification, + $"New {status} finding", + sources); + + _entries.Add(entry); + return this; + } + + /// + /// Adds a resolved finding entry. + /// + public DeltaReportBuilder AddResolved( + string vulnerabilityId, + string productKey, + VexStatus fromStatus, + VexStatus toStatus, + double fromConfidence, + double toConfidence, + VexJustification? justification = null, + IEnumerable? sources = null) + { + var entry = CreateEntry( + vulnerabilityId, + productKey, + DeltaSection.Resolved, + fromStatus, + toStatus, + fromConfidence, + toConfidence, + null, + null, + justification, + $"Resolved: {fromStatus} -> {toStatus}", + sources); + + _entries.Add(entry); + return this; + } + + /// + /// Adds a confidence change entry. + /// + public DeltaReportBuilder AddConfidenceChange( + string vulnerabilityId, + string productKey, + VexStatus status, + double fromConfidence, + double toConfidence, + IEnumerable? sources = null) + { + var delta = toConfidence - fromConfidence; + var section = delta > 0 ? DeltaSection.ConfidenceUp : DeltaSection.ConfidenceDown; + + if (Math.Abs(delta) < _options.ConfidenceChangeThreshold) + { + return this; // Below threshold, don't add + } + + var entry = CreateEntry( + vulnerabilityId, + productKey, + section, + status, + status, + fromConfidence, + toConfidence, + null, + null, + null, + string.Create(CultureInfo.InvariantCulture, + $"Confidence {(delta > 0 ? "increased" : "decreased")}: {fromConfidence:P0} -> {toConfidence:P0}"), + sources); + + _entries.Add(entry); + return this; + } + + /// + /// Adds a policy impact entry. + /// + public DeltaReportBuilder AddPolicyImpact( + string vulnerabilityId, + string productKey, + VexStatus status, + double confidence, + string impactDescription, + IEnumerable? sources = null) + { + var entry = CreateEntry( + vulnerabilityId, + productKey, + DeltaSection.PolicyImpact, + status, + status, + confidence, + confidence, + null, + null, + null, + $"Policy impact: {impactDescription}", + sources); + + _entries.Add(entry); + return this; + } + + /// + /// Adds a damped (suppressed) entry. + /// + public DeltaReportBuilder AddDamped( + string vulnerabilityId, + string productKey, + VexStatus fromStatus, + VexStatus toStatus, + double fromConfidence, + double toConfidence, + string dampReason) + { + if (!_options.IncludeDamped) + { + return this; + } + + var entry = CreateEntry( + vulnerabilityId, + productKey, + DeltaSection.Damped, + fromStatus, + toStatus, + fromConfidence, + toConfidence, + null, + null, + null, + $"Damped: {dampReason}", + null); + + _entries.Add(entry); + return this; + } + + /// + /// Adds an evidence change entry. + /// + public DeltaReportBuilder AddEvidenceChange( + string vulnerabilityId, + string productKey, + VexStatus status, + double confidence, + string fromRationaleClass, + string toRationaleClass, + IEnumerable? sources = null) + { + if (!_options.IncludeEvidenceChanges) + { + return this; + } + + var entry = CreateEntry( + vulnerabilityId, + productKey, + DeltaSection.EvidenceChanged, + status, + status, + confidence, + confidence, + fromRationaleClass, + toRationaleClass, + null, + $"Evidence changed: {fromRationaleClass} -> {toRationaleClass}", + sources); + + _entries.Add(entry); + return this; + } + + /// + /// Builds the delta report. + /// + public DeltaReport Build() + { + var now = _timeProvider.GetUtcNow(); + var reportId = GenerateReportId(now); + + // Sort entries for deterministic output + var sortedEntries = _entries + .OrderBy(e => (int)e.Section) + .ThenBy(e => e.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(e => e.ProductKey, StringComparer.Ordinal) + .ToImmutableArray(); + + return new DeltaReport + { + ReportId = reportId, + FromSnapshotDigest = _fromDigest, + ToSnapshotDigest = _toDigest, + GeneratedAt = now, + Entries = sortedEntries, + Summary = DeltaSummary.FromEntries(sortedEntries) + }; + } + + private DeltaEntry CreateEntry( + string vulnerabilityId, + string productKey, + DeltaSection section, + VexStatus? fromStatus, + VexStatus toStatus, + double? fromConfidence, + double toConfidence, + string? fromRationaleClass, + string? toRationaleClass, + VexJustification? justification, + string summary, + IEnumerable? sources) + { + var now = _timeProvider.GetUtcNow(); + var deltaId = ComputeDeltaId(vulnerabilityId, productKey, section, now); + + return new DeltaEntry + { + DeltaId = deltaId, + VulnerabilityId = vulnerabilityId, + ProductKey = productKey, + Section = section, + FromStatus = fromStatus, + ToStatus = toStatus, + FromConfidence = fromConfidence, + ToConfidence = toConfidence, + FromRationaleClass = fromRationaleClass, + ToRationaleClass = toRationaleClass, + Justification = justification, + Summary = summary, + Timestamp = now, + ContributingSources = sources?.ToImmutableArray() ?? [] + }; + } + + private static string ComputeDeltaId( + string vulnerabilityId, + string productKey, + DeltaSection section, + DateTimeOffset timestamp) + { + var input = $"{vulnerabilityId}|{productKey}|{section}|{timestamp:O}"; + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return Convert.ToHexStringLower(hash)[..16]; + } + + private string GenerateReportId(DateTimeOffset timestamp) + { + var input = $"{_fromDigest}|{_toDigest}|{timestamp:O}"; + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return $"delta-{Convert.ToHexStringLower(hash)[..12]}"; + } +} diff --git a/src/VexLens/StellaOps.VexLens/Delta/DeltaSection.cs b/src/VexLens/StellaOps.VexLens/Delta/DeltaSection.cs new file mode 100644 index 000000000..3c243d367 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Delta/DeltaSection.cs @@ -0,0 +1,86 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Text.Json.Serialization; + +namespace StellaOps.VexLens.Delta; + +/// +/// Categorizes a delta entry for UI presentation. +/// +/// +/// Delta sections enable the UI to present changes in a structured way: +/// - New: First-time findings that require attention +/// - Resolved: Issues that are now fixed or determined not to affect +/// - ConfidenceUp/Down: Changes in certainty that may affect prioritization +/// - PolicyImpact: Changes that affect gate decisions or workflow +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum DeltaSection +{ + /// + /// A new finding that was not present in the previous snapshot. + /// + /// + /// Use when: Status was not present or was under_investigation + /// and is now affected. + /// + [JsonPropertyName("new")] + New, + + /// + /// A finding that has been resolved (no longer actionable). + /// + /// + /// Use when: Status changed from affected to not_affected or fixed. + /// + [JsonPropertyName("resolved")] + Resolved, + + /// + /// Confidence in an existing finding has increased significantly. + /// + /// + /// Use when: Same status but confidence increased by threshold amount. + /// + [JsonPropertyName("confidence_up")] + ConfidenceUp, + + /// + /// Confidence in an existing finding has decreased significantly. + /// + /// + /// Use when: Same status but confidence decreased by threshold amount. + /// + [JsonPropertyName("confidence_down")] + ConfidenceDown, + + /// + /// A change that affects policy gate decisions. + /// + /// + /// Use when: Gate decision changed (pass -> fail, warn -> block, etc.) + /// even if underlying status/confidence didn't change significantly. + /// + [JsonPropertyName("policy_impact")] + PolicyImpact, + + /// + /// A finding that was damped and not surfaced. + /// + /// + /// Use when: Change would normally surface but was suppressed by + /// stability damping. Only included when verbose mode is enabled. + /// + [JsonPropertyName("damped")] + Damped, + + /// + /// A finding where the rationale class changed. + /// + /// + /// Use when: Evidence authority changed (e.g., heuristic -> authoritative) + /// even if status didn't change. + /// + [JsonPropertyName("evidence_changed")] + EvidenceChanged +} diff --git a/src/VexLens/StellaOps.VexLens/Extensions/VexLensServiceCollectionExtensions.cs b/src/VexLens/StellaOps.VexLens/Extensions/VexLensServiceCollectionExtensions.cs index 098a5cf69..e83b00b77 100644 --- a/src/VexLens/StellaOps.VexLens/Extensions/VexLensServiceCollectionExtensions.cs +++ b/src/VexLens/StellaOps.VexLens/Extensions/VexLensServiceCollectionExtensions.cs @@ -4,16 +4,19 @@ using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Npgsql; +using StellaOps.Policy.Engine.Gates; +using StellaOps.ReachGraph.Deduplication; using StellaOps.VexLens.Api; using StellaOps.VexLens.Caching; using StellaOps.VexLens.Consensus; using StellaOps.VexLens.Export; using StellaOps.VexLens.Integration; -using StellaOps.VexLens.Orchestration; using StellaOps.VexLens.Mapping; +using StellaOps.VexLens.NoiseGate; using StellaOps.VexLens.Normalization; using StellaOps.VexLens.Observability; using StellaOps.VexLens.Options; +using StellaOps.VexLens.Orchestration; using StellaOps.VexLens.Storage; using StellaOps.VexLens.Trust; using StellaOps.VexLens.Trust.SourceTrust; @@ -110,6 +113,9 @@ public static class VexLensServiceCollectionExtensions // Consensus engine services.TryAddSingleton(); + // Noise-gating services (Sprint: NG-001) + RegisterNoiseGating(services, options); + // Storage RegisterStorage(services, options.Storage); @@ -292,4 +298,70 @@ public static class VexLensServiceCollectionExtensions dualWriteLogger); }); } + + /// + /// Registers noise-gating services. + /// Sprint: SPRINT_20260104_001_BE_adaptive_noise_gating (NG-001) + /// + private static void RegisterNoiseGating( + IServiceCollection services, + VexLensOptions options) + { + // Configure NoiseGateOptions + services.TryAddSingleton(sp => + { + var noiseGateOptions = new NoiseGateOptions(); + return Microsoft.Extensions.Options.Options.Create(noiseGateOptions); + }); + + // TimeProvider for deterministic time handling + services.TryAddSingleton(TimeProvider.System); + + // Edge deduplication + services.TryAddSingleton(); + + // Stability damping gate + services.TryAddSingleton>(sp => + { + var dampingOptions = new StabilityDampingOptions(); + return new OptionsMonitorAdapter(dampingOptions); + }); + services.TryAddSingleton(); + + // Noise gate service + services.TryAddSingleton>(sp => + { + var opts = sp.GetRequiredService>(); + return new OptionsMonitorAdapter(opts.Value); + }); + services.TryAddSingleton(); + + // Snapshot and statistics storage (Sprint: NG-FE-001) + services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + var timeProvider = sp.GetRequiredService(); + return new InMemoryGatingStatisticsStore(timeProvider); + }); + } +} + +/// +/// Simple adapter to convert IOptions to IOptionsMonitor for singleton services. +/// +internal sealed class OptionsMonitorAdapter : IOptionsMonitor + where T : class +{ + private readonly T _value; + + public OptionsMonitorAdapter(T value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + public T CurrentValue => _value; + + public T Get(string? name) => _value; + + public IDisposable? OnChange(Action listener) => null; } diff --git a/src/VexLens/StellaOps.VexLens/NoiseGate/INoiseGate.cs b/src/VexLens/StellaOps.VexLens/NoiseGate/INoiseGate.cs new file mode 100644 index 000000000..a57f9d0bf --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/NoiseGate/INoiseGate.cs @@ -0,0 +1,310 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using StellaOps.ReachGraph.Deduplication; +using StellaOps.ReachGraph.Schema; +using StellaOps.VexLens.Delta; +using StellaOps.VexLens.Models; + +namespace StellaOps.VexLens.NoiseGate; + +/// +/// Central interface for noise-gating operations on vulnerability graphs. +/// +/// +/// The noise gate provides three core capabilities: +/// +/// Edge deduplication: Collapses semantically equivalent edges from multiple sources +/// Verdict resolution: Applies stability damping to prevent flip-flopping +/// Delta reporting: Computes meaningful changes between snapshots +/// +/// +public interface INoiseGate +{ + /// + /// Deduplicates edges based on semantic equivalence. + /// + /// The edges to deduplicate. + /// Cancellation token. + /// Deduplicated edges with merged provenance. + Task> DedupeEdgesAsync( + IReadOnlyList edges, + CancellationToken cancellationToken = default); + + /// + /// Resolves a verdict by applying stability damping. + /// + /// The verdict resolution request. + /// Cancellation token. + /// The resolved verdict with damping decision. + Task ResolveVerdictAsync( + VerdictResolutionRequest request, + CancellationToken cancellationToken = default); + + /// + /// Applies noise-gating to a graph snapshot. + /// + /// The gating request. + /// Cancellation token. + /// The gated graph snapshot. + Task GateAsync( + NoiseGateRequest request, + CancellationToken cancellationToken = default); + + /// + /// Computes a delta report between two snapshots. + /// + /// The previous snapshot. + /// The current snapshot. + /// Optional report options. + /// Cancellation token. + /// The delta report. + Task DiffAsync( + GatedGraphSnapshot fromSnapshot, + GatedGraphSnapshot toSnapshot, + DeltaReportOptions? options = null, + CancellationToken cancellationToken = default); +} + +/// +/// Request to resolve a verdict with stability damping. +/// +public sealed record VerdictResolutionRequest +{ + /// + /// Gets the unique key for this verdict (e.g., "artifact:cve"). + /// + public required string Key { get; init; } + + /// + /// Gets the vulnerability ID. + /// + public required string VulnerabilityId { get; init; } + + /// + /// Gets the product key (PURL or other identifier). + /// + public required string ProductKey { get; init; } + + /// + /// Gets the proposed VEX status. + /// + public required VexStatus ProposedStatus { get; init; } + + /// + /// Gets the proposed confidence score. + /// + public required double ProposedConfidence { get; init; } + + /// + /// Gets the rationale class for the verdict. + /// + public string? RationaleClass { get; init; } + + /// + /// Gets the justification for the verdict. + /// + public VexJustification? Justification { get; init; } + + /// + /// Gets the contributing sources. + /// + public IReadOnlyList? ContributingSources { get; init; } + + /// + /// Gets the tenant ID for multi-tenant deployments. + /// + public string? TenantId { get; init; } +} + +/// +/// Result of verdict resolution with damping decision. +/// +public sealed record ResolvedVerdict +{ + /// + /// Gets the vulnerability ID. + /// + public required string VulnerabilityId { get; init; } + + /// + /// Gets the product key. + /// + public required string ProductKey { get; init; } + + /// + /// Gets the final VEX status. + /// + public required VexStatus Status { get; init; } + + /// + /// Gets the final confidence score. + /// + public required double Confidence { get; init; } + + /// + /// Gets the rationale class. + /// + public string? RationaleClass { get; init; } + + /// + /// Gets the justification. + /// + public VexJustification? Justification { get; init; } + + /// + /// Gets whether the verdict was surfaced (not damped). + /// + public required bool WasSurfaced { get; init; } + + /// + /// Gets the damping reason if applicable. + /// + public string? DampingReason { get; init; } + + /// + /// Gets the previous status if available. + /// + public VexStatus? PreviousStatus { get; init; } + + /// + /// Gets the previous confidence if available. + /// + public double? PreviousConfidence { get; init; } + + /// + /// Gets the contributing sources. + /// + public ImmutableArray ContributingSources { get; init; } = []; + + /// + /// Gets the timestamp of resolution. + /// + public required DateTimeOffset ResolvedAt { get; init; } +} + +/// +/// Request to apply noise-gating to a graph. +/// +public sealed record NoiseGateRequest +{ + /// + /// Gets the reachability graph to gate. + /// + public required ReachGraphMinimal Graph { get; init; } + + /// + /// Gets the verdicts to include. + /// + public required IReadOnlyList Verdicts { get; init; } + + /// + /// Gets the snapshot ID. + /// + public required string SnapshotId { get; init; } + + /// + /// Gets the tenant ID. + /// + public string? TenantId { get; init; } + + /// + /// Gets whether to compute a previous snapshot diff. + /// + public bool ComputeDiff { get; init; } = true; + + /// + /// Gets the previous snapshot ID for diff computation. + /// + public string? PreviousSnapshotId { get; init; } +} + +/// +/// A gated graph snapshot with deduplicated edges and resolved verdicts. +/// +public sealed record GatedGraphSnapshot +{ + /// + /// Gets the unique snapshot identifier. + /// + public required string SnapshotId { get; init; } + + /// + /// Gets the snapshot digest for integrity verification. + /// + public required string Digest { get; init; } + + /// + /// Gets the artifact this snapshot describes. + /// + public required ReachGraphArtifact Artifact { get; init; } + + /// + /// Gets the deduplicated edges. + /// + public required ImmutableArray Edges { get; init; } + + /// + /// Gets the resolved verdicts. + /// + public required ImmutableArray Verdicts { get; init; } + + /// + /// Gets the verdicts that were damped (not surfaced). + /// + public ImmutableArray DampedVerdicts { get; init; } = []; + + /// + /// Gets the timestamp when this snapshot was created. + /// + public required DateTimeOffset CreatedAt { get; init; } + + /// + /// Gets the gating statistics. + /// + public required GatingStatistics Statistics { get; init; } +} + +/// +/// Statistics from a noise-gating operation. +/// +public sealed record GatingStatistics +{ + /// + /// Gets the original edge count before deduplication. + /// + public required int OriginalEdgeCount { get; init; } + + /// + /// Gets the edge count after deduplication. + /// + public required int DeduplicatedEdgeCount { get; init; } + + /// + /// Gets the edge reduction percentage. + /// + public double EdgeReductionPercent => + OriginalEdgeCount > 0 + ? (1.0 - (double)DeduplicatedEdgeCount / OriginalEdgeCount) * 100.0 + : 0.0; + + /// + /// Gets the total verdict count. + /// + public required int TotalVerdictCount { get; init; } + + /// + /// Gets the surfaced verdict count. + /// + public required int SurfacedVerdictCount { get; init; } + + /// + /// Gets the damped verdict count. + /// + public required int DampedVerdictCount { get; init; } + + /// + /// Gets the duration of the gating operation. + /// + public required TimeSpan Duration { get; init; } +} diff --git a/src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateOptions.cs b/src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateOptions.cs new file mode 100644 index 000000000..41e15f350 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateOptions.cs @@ -0,0 +1,122 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +namespace StellaOps.VexLens.NoiseGate; + +/// +/// Configuration options for noise-gating in VexLens. +/// +public sealed class NoiseGateOptions +{ + /// + /// Configuration section name. + /// + public const string SectionName = "VexLens:NoiseGate"; + + /// + /// Gets or sets whether noise-gating is enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Gets or sets whether edge deduplication is enabled. + /// + public bool EdgeDeduplicationEnabled { get; set; } = true; + + /// + /// Gets or sets whether stability damping is enabled. + /// + public bool StabilityDampingEnabled { get; set; } = true; + + /// + /// Gets or sets whether delta reports should include damped entries. + /// + public bool IncludeDampedInDelta { get; set; } = false; + + /// + /// Gets or sets the minimum confidence threshold for including verdicts. + /// Verdicts below this threshold are excluded from output. + /// + public double MinConfidenceThreshold { get; set; } = 0.0; + + /// + /// Gets or sets the confidence change threshold for triggering delta sections. + /// + public double ConfidenceChangeThreshold { get; set; } = 0.15; + + /// + /// Gets or sets whether to collapse semantically equivalent edges. + /// + public bool CollapseEquivalentEdges { get; set; } = true; + + /// + /// Gets or sets the maximum number of edges to process per operation. + /// + public int MaxEdgesPerOperation { get; set; } = 100_000; + + /// + /// Gets or sets the maximum number of verdicts to process per operation. + /// + public int MaxVerdictsPerOperation { get; set; } = 50_000; + + /// + /// Gets or sets whether to log noise-gating decisions. + /// + public bool LogDecisions { get; set; } = false; + + /// + /// Gets or sets the snapshot retention for delta computation. + /// + public TimeSpan SnapshotRetention { get; set; } = TimeSpan.FromDays(30); + + /// + /// Gets or sets options specific to edge deduplication. + /// + public EdgeDeduplicationOptions EdgeDeduplication { get; set; } = new(); + + /// + /// Gets or sets options specific to stability damping. + /// + public StabilityDampingSettings StabilityDamping { get; set; } = new(); +} + +/// +/// Options for edge deduplication. +/// +public sealed class EdgeDeduplicationOptions +{ + /// + /// Gets or sets the minimum provenance count to consider an edge reliable. + /// + public int MinProvenanceCount { get; set; } = 1; + + /// + /// Gets or sets whether to merge provenance from deduplicated edges. + /// + public bool MergeProvenance { get; set; } = true; + + /// + /// Gets or sets whether to use the highest confidence from merged edges. + /// + public bool UseHighestConfidence { get; set; } = true; +} + +/// +/// Settings for stability damping within noise-gating. +/// +public sealed class StabilityDampingSettings +{ + /// + /// Gets or sets the minimum duration before a state change is surfaced. + /// + public TimeSpan MinDurationBeforeChange { get; set; } = TimeSpan.FromHours(4); + + /// + /// Gets or sets the minimum confidence delta for immediate surfacing. + /// + public double MinConfidenceDeltaPercent { get; set; } = 0.15; + + /// + /// Gets or sets whether to only damp downgrades (not upgrades). + /// + public bool OnlyDampDowngrades { get; set; } = true; +} diff --git a/src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateService.cs b/src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateService.cs new file mode 100644 index 000000000..d7072f41c --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/NoiseGate/NoiseGateService.cs @@ -0,0 +1,471 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using System.Diagnostics; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Policy.Engine.Gates; +using StellaOps.ReachGraph.Deduplication; +using StellaOps.ReachGraph.Schema; +using StellaOps.VexLens.Delta; +using StellaOps.VexLens.Models; + +namespace StellaOps.VexLens.NoiseGate; + +/// +/// Implementation of that integrates edge deduplication, +/// stability damping, and delta report generation. +/// +public sealed class NoiseGateService : INoiseGate +{ + private readonly IEdgeDeduplicator _edgeDeduplicator; + private readonly IStabilityDampingGate _stabilityDampingGate; + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + /// + /// Initializes a new instance of the class. + /// + public NoiseGateService( + IEdgeDeduplicator edgeDeduplicator, + IStabilityDampingGate stabilityDampingGate, + IOptionsMonitor options, + TimeProvider timeProvider, + ILogger logger) + { + _edgeDeduplicator = edgeDeduplicator ?? throw new ArgumentNullException(nameof(edgeDeduplicator)); + _stabilityDampingGate = stabilityDampingGate ?? throw new ArgumentNullException(nameof(stabilityDampingGate)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + public Task> DedupeEdgesAsync( + IReadOnlyList edges, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(edges); + + var opts = _options.CurrentValue; + + if (!opts.EdgeDeduplicationEnabled || !opts.Enabled) + { + // Return edges without deduplication - create minimal deduplicated wrappers + var passthrough = edges.Select(e => new DeduplicatedEdgeBuilder( + e.From, e.To, e.Why.Type, e.Why.Loc) + .WithConfidence(e.Why.Confidence) + .Build()) + .ToList(); + + return Task.FromResult>(passthrough); + } + + var result = _edgeDeduplicator.Deduplicate(edges); + return Task.FromResult(result); + } + + /// + public async Task ResolveVerdictAsync( + VerdictResolutionRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var opts = _options.CurrentValue; + var now = _timeProvider.GetUtcNow(); + + // If stability damping is disabled, pass through + if (!opts.StabilityDampingEnabled || !opts.Enabled) + { + return new ResolvedVerdict + { + VulnerabilityId = request.VulnerabilityId, + ProductKey = request.ProductKey, + Status = request.ProposedStatus, + Confidence = request.ProposedConfidence, + RationaleClass = request.RationaleClass, + Justification = request.Justification, + WasSurfaced = true, + DampingReason = null, + ContributingSources = request.ContributingSources?.ToImmutableArray() ?? [], + ResolvedAt = now + }; + } + + // Evaluate with stability damping gate + var dampingRequest = new StabilityDampingRequest + { + Key = request.Key, + TenantId = request.TenantId, + ProposedState = new VerdictState + { + Status = VexStatusToString(request.ProposedStatus), + Confidence = request.ProposedConfidence, + Timestamp = now, + RationaleClass = request.RationaleClass, + SourceId = request.ContributingSources?.FirstOrDefault() + } + }; + + var decision = await _stabilityDampingGate.EvaluateAsync(dampingRequest, cancellationToken) + .ConfigureAwait(false); + + // Record the new state if surfaced + if (decision.ShouldSurface) + { + await _stabilityDampingGate.RecordStateAsync( + request.Key, + dampingRequest.ProposedState, + cancellationToken).ConfigureAwait(false); + } + + if (opts.LogDecisions) + { + _logger.LogDebug( + "Verdict resolution for {Key}: surfaced={Surfaced}, reason={Reason}", + request.Key, + decision.ShouldSurface, + decision.Reason); + } + + return new ResolvedVerdict + { + VulnerabilityId = request.VulnerabilityId, + ProductKey = request.ProductKey, + Status = request.ProposedStatus, + Confidence = request.ProposedConfidence, + RationaleClass = request.RationaleClass, + Justification = request.Justification, + WasSurfaced = decision.ShouldSurface, + DampingReason = decision.ShouldSurface ? null : decision.Reason, + PreviousStatus = decision.PreviousState != null + ? ParseVexStatus(decision.PreviousState.Status) + : null, + PreviousConfidence = decision.PreviousState?.Confidence, + ContributingSources = request.ContributingSources?.ToImmutableArray() ?? [], + ResolvedAt = now + }; + } + + /// + public async Task GateAsync( + NoiseGateRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var opts = _options.CurrentValue; + var now = _timeProvider.GetUtcNow(); + var stopwatch = Stopwatch.StartNew(); + + // Validate limits + if (request.Graph.Edges.Length > opts.MaxEdgesPerOperation) + { + throw new InvalidOperationException(string.Create(CultureInfo.InvariantCulture, + $"Edge count {request.Graph.Edges.Length} exceeds maximum {opts.MaxEdgesPerOperation}")); + } + + if (request.Verdicts.Count > opts.MaxVerdictsPerOperation) + { + throw new InvalidOperationException(string.Create(CultureInfo.InvariantCulture, + $"Verdict count {request.Verdicts.Count} exceeds maximum {opts.MaxVerdictsPerOperation}")); + } + + // Deduplicate edges + var edges = request.Graph.Edges.ToList(); + var deduplicatedEdges = await DedupeEdgesAsync(edges, cancellationToken).ConfigureAwait(false); + + // Resolve verdicts + var surfacedVerdicts = new List(); + var dampedVerdicts = new List(); + + foreach (var verdictRequest in request.Verdicts) + { + var resolved = await ResolveVerdictAsync(verdictRequest, cancellationToken).ConfigureAwait(false); + + if (resolved.WasSurfaced) + { + // Apply confidence threshold + if (resolved.Confidence >= opts.MinConfidenceThreshold) + { + surfacedVerdicts.Add(resolved); + } + else if (opts.IncludeDampedInDelta) + { + dampedVerdicts.Add(resolved with + { + WasSurfaced = false, + DampingReason = string.Create(CultureInfo.InvariantCulture, + $"Confidence {resolved.Confidence:P1} below threshold {opts.MinConfidenceThreshold:P1}") + }); + } + } + else + { + dampedVerdicts.Add(resolved); + } + } + + stopwatch.Stop(); + + // Compute snapshot digest + var digest = ComputeSnapshotDigest( + request.SnapshotId, + deduplicatedEdges, + surfacedVerdicts, + now); + + var statistics = new GatingStatistics + { + OriginalEdgeCount = edges.Count, + DeduplicatedEdgeCount = deduplicatedEdges.Count, + TotalVerdictCount = request.Verdicts.Count, + SurfacedVerdictCount = surfacedVerdicts.Count, + DampedVerdictCount = dampedVerdicts.Count, + Duration = stopwatch.Elapsed + }; + + _logger.LogInformation( + "Gated snapshot {SnapshotId}: edges {OriginalEdges}->{DeduplicatedEdges} ({Reduction:F1}% reduction), " + + "verdicts {Surfaced}/{Total} surfaced, {Damped} damped in {Duration}ms", + request.SnapshotId, + statistics.OriginalEdgeCount, + statistics.DeduplicatedEdgeCount, + statistics.EdgeReductionPercent, + statistics.SurfacedVerdictCount, + statistics.TotalVerdictCount, + statistics.DampedVerdictCount, + statistics.Duration.TotalMilliseconds); + + return new GatedGraphSnapshot + { + SnapshotId = request.SnapshotId, + Digest = digest, + Artifact = request.Graph.Artifact, + Edges = deduplicatedEdges.ToImmutableArray(), + Verdicts = surfacedVerdicts.ToImmutableArray(), + DampedVerdicts = opts.IncludeDampedInDelta + ? dampedVerdicts.ToImmutableArray() + : [], + CreatedAt = now, + Statistics = statistics + }; + } + + /// + public Task DiffAsync( + GatedGraphSnapshot fromSnapshot, + GatedGraphSnapshot toSnapshot, + DeltaReportOptions? options = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(fromSnapshot); + ArgumentNullException.ThrowIfNull(toSnapshot); + + var opts = _options.CurrentValue; + var reportOptions = options ?? new DeltaReportOptions + { + ConfidenceChangeThreshold = opts.ConfidenceChangeThreshold, + IncludeDamped = opts.IncludeDampedInDelta, + IncludeEvidenceChanges = true + }; + + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots(fromSnapshot.Digest, toSnapshot.Digest) + .WithOptions(reportOptions); + + // Index previous verdicts by key + var previousVerdicts = fromSnapshot.Verdicts + .ToDictionary( + v => $"{v.VulnerabilityId}|{v.ProductKey}", + v => v, + StringComparer.Ordinal); + + // Process current verdicts + foreach (var current in toSnapshot.Verdicts) + { + var key = $"{current.VulnerabilityId}|{current.ProductKey}"; + + if (!previousVerdicts.TryGetValue(key, out var previous)) + { + // New finding + builder.AddNew( + current.VulnerabilityId, + current.ProductKey, + current.Status, + current.Confidence, + current.RationaleClass, + current.Justification, + current.ContributingSources); + } + else + { + // Existing finding - check for changes + previousVerdicts.Remove(key); // Mark as processed + + var statusChanged = current.Status != previous.Status; + var confidenceChanged = Math.Abs(current.Confidence - previous.Confidence) >= reportOptions.ConfidenceChangeThreshold; + var rationaleChanged = !string.Equals(current.RationaleClass, previous.RationaleClass, StringComparison.Ordinal); + + if (statusChanged) + { + // Check if resolved + if (IsResolved(current.Status) && !IsResolved(previous.Status)) + { + builder.AddResolved( + current.VulnerabilityId, + current.ProductKey, + previous.Status, + current.Status, + previous.Confidence, + current.Confidence, + current.Justification, + current.ContributingSources); + } + else + { + // Status change but not resolved - treat as policy impact + builder.AddPolicyImpact( + current.VulnerabilityId, + current.ProductKey, + current.Status, + current.Confidence, + string.Create(CultureInfo.InvariantCulture, + $"Status changed: {previous.Status} -> {current.Status}"), + current.ContributingSources); + } + } + else if (confidenceChanged) + { + builder.AddConfidenceChange( + current.VulnerabilityId, + current.ProductKey, + current.Status, + previous.Confidence, + current.Confidence, + current.ContributingSources); + } + else if (rationaleChanged && !string.IsNullOrEmpty(current.RationaleClass)) + { + builder.AddEvidenceChange( + current.VulnerabilityId, + current.ProductKey, + current.Status, + current.Confidence, + previous.RationaleClass ?? "unknown", + current.RationaleClass, + current.ContributingSources); + } + } + } + + // Remaining previous verdicts are no longer present - they resolved + foreach (var (key, previous) in previousVerdicts) + { + // Treat as resolved (no longer affected) + builder.AddResolved( + previous.VulnerabilityId, + previous.ProductKey, + previous.Status, + VexStatus.NotAffected, // Assumed resolved + previous.Confidence, + 1.0, // High confidence in removal + VexJustification.VulnerableCodeNotPresent, + null); + } + + // Add damped entries if configured + if (reportOptions.IncludeDamped) + { + foreach (var damped in toSnapshot.DampedVerdicts) + { + if (damped.PreviousStatus.HasValue) + { + builder.AddDamped( + damped.VulnerabilityId, + damped.ProductKey, + damped.PreviousStatus.Value, + damped.Status, + damped.PreviousConfidence ?? 0.0, + damped.Confidence, + damped.DampingReason ?? "Unknown"); + } + } + } + + return Task.FromResult(builder.Build()); + } + + private static bool IsResolved(VexStatus status) => + status == VexStatus.NotAffected || status == VexStatus.Fixed; + + private static string VexStatusToString(VexStatus status) => status switch + { + VexStatus.Affected => "affected", + VexStatus.NotAffected => "not_affected", + VexStatus.Fixed => "fixed", + VexStatus.UnderInvestigation => "under_investigation", + _ => "unknown" + }; + + private static VexStatus? ParseVexStatus(string status) => + status?.ToLowerInvariant() switch + { + "affected" => VexStatus.Affected, + "not_affected" => VexStatus.NotAffected, + "fixed" => VexStatus.Fixed, + "under_investigation" => VexStatus.UnderInvestigation, + _ => null + }; + + private static string ComputeSnapshotDigest( + string snapshotId, + IReadOnlyList edges, + IReadOnlyList verdicts, + DateTimeOffset timestamp) + { + // Build deterministic input for digest + var sb = new StringBuilder(); + sb.Append(snapshotId); + sb.Append('|'); + sb.Append(timestamp.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)); + + // Add sorted edges + var sortedEdges = edges + .OrderBy(e => e.SemanticKey, StringComparer.Ordinal) + .ToList(); + + foreach (var edge in sortedEdges) + { + sb.Append('|'); + sb.Append(edge.SemanticKey); + } + + // Add sorted verdicts + var sortedVerdicts = verdicts + .OrderBy(v => v.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(v => v.ProductKey, StringComparer.Ordinal) + .ToList(); + + foreach (var verdict in sortedVerdicts) + { + sb.Append('|'); + sb.Append(verdict.VulnerabilityId); + sb.Append(':'); + sb.Append(verdict.ProductKey); + sb.Append(':'); + sb.Append(VexStatusToString(verdict.Status)); + sb.Append(':'); + sb.Append(verdict.Confidence.ToString("F4", CultureInfo.InvariantCulture)); + } + + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(sb.ToString())); + return $"sha256:{Convert.ToHexStringLower(hash)}"; + } +} diff --git a/src/VexLens/StellaOps.VexLens/StellaOps.VexLens.csproj b/src/VexLens/StellaOps.VexLens/StellaOps.VexLens.csproj index 0956b93d0..ad459b71f 100644 --- a/src/VexLens/StellaOps.VexLens/StellaOps.VexLens.csproj +++ b/src/VexLens/StellaOps.VexLens/StellaOps.VexLens.csproj @@ -27,6 +27,9 @@ + + + diff --git a/src/VexLens/StellaOps.VexLens/Storage/IGatingStatisticsStore.cs b/src/VexLens/StellaOps.VexLens/Storage/IGatingStatisticsStore.cs new file mode 100644 index 000000000..5ccec2280 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Storage/IGatingStatisticsStore.cs @@ -0,0 +1,99 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using StellaOps.VexLens.NoiseGate; + +namespace StellaOps.VexLens.Storage; + +/// +/// Store for aggregated gating statistics. +/// +public interface IGatingStatisticsStore +{ + /// + /// Records statistics from a gating operation. + /// + /// The snapshot ID. + /// The gating statistics. + /// Optional tenant ID. + /// Cancellation token. + Task RecordAsync( + string snapshotId, + GatingStatistics statistics, + string? tenantId = null, + CancellationToken cancellationToken = default); + + /// + /// Gets aggregated statistics for a time range. + /// + /// Optional tenant ID. + /// Start date filter. + /// End date filter. + /// Cancellation token. + /// Aggregated statistics. + Task GetAggregatedAsync( + string? tenantId = null, + DateTimeOffset? fromDate = null, + DateTimeOffset? toDate = null, + CancellationToken cancellationToken = default); +} + +/// +/// Aggregated gating statistics across multiple snapshots. +/// +public sealed record AggregatedGatingStatistics +{ + /// + /// Gets the total number of snapshots processed. + /// + public required int TotalSnapshots { get; init; } + + /// + /// Gets the total edges processed. + /// + public required int TotalEdgesProcessed { get; init; } + + /// + /// Gets the total edges after deduplication. + /// + public required int TotalEdgesAfterDedup { get; init; } + + /// + /// Gets the average edge reduction percentage. + /// + public required double AverageEdgeReductionPercent { get; init; } + + /// + /// Gets the total verdicts processed. + /// + public required int TotalVerdicts { get; init; } + + /// + /// Gets the total surfaced verdicts. + /// + public required int TotalSurfaced { get; init; } + + /// + /// Gets the total damped verdicts. + /// + public required int TotalDamped { get; init; } + + /// + /// Gets the average damping percentage. + /// + public required double AverageDampingPercent { get; init; } + + /// + /// Empty statistics for when no data exists. + /// + public static AggregatedGatingStatistics Empty => new() + { + TotalSnapshots = 0, + TotalEdgesProcessed = 0, + TotalEdgesAfterDedup = 0, + AverageEdgeReductionPercent = 0, + TotalVerdicts = 0, + TotalSurfaced = 0, + TotalDamped = 0, + AverageDampingPercent = 0 + }; +} diff --git a/src/VexLens/StellaOps.VexLens/Storage/ISnapshotStore.cs b/src/VexLens/StellaOps.VexLens/Storage/ISnapshotStore.cs new file mode 100644 index 000000000..932ab6182 --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Storage/ISnapshotStore.cs @@ -0,0 +1,96 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using StellaOps.ReachGraph.Schema; +using StellaOps.VexLens.NoiseGate; + +namespace StellaOps.VexLens.Storage; + +/// +/// Store for managing graph snapshots (both raw and gated). +/// +public interface ISnapshotStore +{ + /// + /// Gets a gated snapshot by ID. + /// + /// The snapshot ID. + /// Optional tenant ID. + /// Cancellation token. + /// The gated snapshot, or null if not found. + Task GetAsync( + string snapshotId, + string? tenantId = null, + CancellationToken cancellationToken = default); + + /// + /// Gets a raw (ungated) snapshot by ID. + /// + /// The snapshot ID. + /// Optional tenant ID. + /// Cancellation token. + /// The raw snapshot, or null if not found. + Task GetRawAsync( + string snapshotId, + string? tenantId = null, + CancellationToken cancellationToken = default); + + /// + /// Stores a gated snapshot. + /// + /// The snapshot to store. + /// Optional tenant ID. + /// Cancellation token. + Task StoreAsync( + GatedGraphSnapshot snapshot, + string? tenantId = null, + CancellationToken cancellationToken = default); + + /// + /// Stores a raw snapshot. + /// + /// The raw snapshot to store. + /// Optional tenant ID. + /// Cancellation token. + Task StoreRawAsync( + RawGraphSnapshot snapshot, + string? tenantId = null, + CancellationToken cancellationToken = default); + + /// + /// Lists snapshot IDs for a tenant. + /// + /// Optional tenant ID. + /// Maximum number of results. + /// Cancellation token. + /// List of snapshot IDs. + Task> ListAsync( + string? tenantId = null, + int limit = 100, + CancellationToken cancellationToken = default); +} + +/// +/// A raw (ungated) graph snapshot. +/// +public sealed record RawGraphSnapshot +{ + /// + /// Gets the snapshot ID. + /// + public required string SnapshotId { get; init; } + + /// + /// Gets the reachability graph. + /// + public required ReachGraphMinimal Graph { get; init; } + + /// + /// Gets the verdict requests. + /// + public required IReadOnlyList Verdicts { get; init; } + + /// + /// Gets the timestamp when this snapshot was created. + /// + public required DateTimeOffset CreatedAt { get; init; } +} diff --git a/src/VexLens/StellaOps.VexLens/Storage/InMemoryGatingStatisticsStore.cs b/src/VexLens/StellaOps.VexLens/Storage/InMemoryGatingStatisticsStore.cs new file mode 100644 index 000000000..c65a6fc3f --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Storage/InMemoryGatingStatisticsStore.cs @@ -0,0 +1,89 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Concurrent; +using System.Globalization; +using StellaOps.VexLens.NoiseGate; + +namespace StellaOps.VexLens.Storage; + +/// +/// In-memory implementation of for development and testing. +/// +public sealed class InMemoryGatingStatisticsStore : IGatingStatisticsStore +{ + private readonly ConcurrentDictionary _entries = new(); + private readonly TimeProvider _timeProvider; + + /// + /// Initializes a new instance of in-memory statistics store. + /// + public InMemoryGatingStatisticsStore(TimeProvider? timeProvider = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public Task RecordAsync( + string snapshotId, + GatingStatistics statistics, + string? tenantId = null, + CancellationToken cancellationToken = default) + { + var key = MakeKey(snapshotId, tenantId); + _entries[key] = new StatisticsEntry(statistics, _timeProvider.GetUtcNow()); + return Task.CompletedTask; + } + + /// + public Task GetAggregatedAsync( + string? tenantId = null, + DateTimeOffset? fromDate = null, + DateTimeOffset? toDate = null, + CancellationToken cancellationToken = default) + { + var prefix = tenantId is null ? string.Empty : $"{tenantId}:"; + + var entries = _entries + .Where(kvp => string.IsNullOrEmpty(prefix) || kvp.Key.StartsWith(prefix, StringComparison.Ordinal)) + .Where(kvp => fromDate is null || kvp.Value.RecordedAt >= fromDate) + .Where(kvp => toDate is null || kvp.Value.RecordedAt <= toDate) + .Select(kvp => kvp.Value.Statistics) + .ToList(); + + if (entries.Count == 0) + { + return Task.FromResult(AggregatedGatingStatistics.Empty); + } + + var totalEdgesProcessed = entries.Sum(s => s.OriginalEdgeCount); + var totalEdgesAfterDedup = entries.Sum(s => s.DeduplicatedEdgeCount); + var totalVerdicts = entries.Sum(s => s.TotalVerdictCount); + var totalSurfaced = entries.Sum(s => s.SurfacedVerdictCount); + var totalDamped = entries.Sum(s => s.DampedVerdictCount); + + var avgEdgeReduction = totalEdgesProcessed > 0 + ? (1.0 - (double)totalEdgesAfterDedup / totalEdgesProcessed) * 100.0 + : 0.0; + + var avgDampingPercent = totalVerdicts > 0 + ? (double)totalDamped / totalVerdicts * 100.0 + : 0.0; + + return Task.FromResult(new AggregatedGatingStatistics + { + TotalSnapshots = entries.Count, + TotalEdgesProcessed = totalEdgesProcessed, + TotalEdgesAfterDedup = totalEdgesAfterDedup, + AverageEdgeReductionPercent = avgEdgeReduction, + TotalVerdicts = totalVerdicts, + TotalSurfaced = totalSurfaced, + TotalDamped = totalDamped, + AverageDampingPercent = avgDampingPercent + }); + } + + private static string MakeKey(string snapshotId, string? tenantId) => + tenantId is null ? snapshotId : $"{tenantId}:{snapshotId}"; + + private sealed record StatisticsEntry(GatingStatistics Statistics, DateTimeOffset RecordedAt); +} diff --git a/src/VexLens/StellaOps.VexLens/Storage/InMemorySnapshotStore.cs b/src/VexLens/StellaOps.VexLens/Storage/InMemorySnapshotStore.cs new file mode 100644 index 000000000..2e893884d --- /dev/null +++ b/src/VexLens/StellaOps.VexLens/Storage/InMemorySnapshotStore.cs @@ -0,0 +1,83 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Concurrent; +using StellaOps.VexLens.NoiseGate; + +namespace StellaOps.VexLens.Storage; + +/// +/// In-memory implementation of for development and testing. +/// +public sealed class InMemorySnapshotStore : ISnapshotStore +{ + private readonly ConcurrentDictionary _gated = new(); + private readonly ConcurrentDictionary _raw = new(); + private readonly ConcurrentDictionary _timestamps = new(); + + /// + public Task GetAsync( + string snapshotId, + string? tenantId = null, + CancellationToken cancellationToken = default) + { + var key = MakeKey(snapshotId, tenantId); + _gated.TryGetValue(key, out var snapshot); + return Task.FromResult(snapshot); + } + + /// + public Task GetRawAsync( + string snapshotId, + string? tenantId = null, + CancellationToken cancellationToken = default) + { + var key = MakeKey(snapshotId, tenantId); + _raw.TryGetValue(key, out var snapshot); + return Task.FromResult(snapshot); + } + + /// + public Task StoreAsync( + GatedGraphSnapshot snapshot, + string? tenantId = null, + CancellationToken cancellationToken = default) + { + var key = MakeKey(snapshot.SnapshotId, tenantId); + _gated[key] = snapshot; + _timestamps[key] = snapshot.CreatedAt; + return Task.CompletedTask; + } + + /// + public Task StoreRawAsync( + RawGraphSnapshot snapshot, + string? tenantId = null, + CancellationToken cancellationToken = default) + { + var key = MakeKey(snapshot.SnapshotId, tenantId); + _raw[key] = snapshot; + _timestamps[key] = snapshot.CreatedAt; + return Task.CompletedTask; + } + + /// + public Task> ListAsync( + string? tenantId = null, + int limit = 100, + CancellationToken cancellationToken = default) + { + var prefix = tenantId is null ? string.Empty : $"{tenantId}:"; + + var ids = _gated.Keys + .Where(k => string.IsNullOrEmpty(prefix) || k.StartsWith(prefix, StringComparison.Ordinal)) + .Select(k => string.IsNullOrEmpty(prefix) ? k : k[prefix.Length..]) + .OrderByDescending(id => _timestamps.TryGetValue(MakeKey(id, tenantId), out var ts) ? ts : DateTimeOffset.MinValue) + .Take(limit) + .ToList(); + + return Task.FromResult>(ids); + } + + private static string MakeKey(string snapshotId, string? tenantId) => + tenantId is null ? snapshotId : $"{tenantId}:{snapshotId}"; +} diff --git a/src/VexLens/__Tests/StellaOps.VexLens.Tests/Delta/DeltaReportBuilderTests.cs b/src/VexLens/__Tests/StellaOps.VexLens.Tests/Delta/DeltaReportBuilderTests.cs new file mode 100644 index 000000000..be591bded --- /dev/null +++ b/src/VexLens/__Tests/StellaOps.VexLens.Tests/Delta/DeltaReportBuilderTests.cs @@ -0,0 +1,364 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using FluentAssertions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.VexLens.Delta; +using StellaOps.VexLens.Models; +using Xunit; + +namespace StellaOps.VexLens.Tests.Delta; + +/// +/// Unit tests for . +/// +[Trait("Category", "Unit")] +public class DeltaReportBuilderTests +{ + private readonly FakeTimeProvider _timeProvider; + + public DeltaReportBuilderTests() + { + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public void Build_EmptyReport_ShouldHaveZeroCounts() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to"); + + // Act + var report = builder.Build(); + + // Assert + report.Summary.TotalCount.Should().Be(0); + report.Summary.NewCount.Should().Be(0); + report.Summary.ResolvedCount.Should().Be(0); + report.HasActionableChanges.Should().BeFalse(); + } + + [Fact] + public void AddNew_ShouldAddNewEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to"); + + // Act + builder.AddNew( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + 0.85, + "binary", + null, + ["nvd", "github"]); + + var report = builder.Build(); + + // Assert + report.Summary.NewCount.Should().Be(1); + report.HasActionableChanges.Should().BeTrue(); + report.GetSection(DeltaSection.New).Should().HaveCount(1); + + var entry = report.GetSection(DeltaSection.New)[0]; + entry.VulnerabilityId.Should().Be("CVE-2024-1234"); + entry.ProductKey.Should().Be("pkg:npm/lodash@4.17.21"); + entry.ToStatus.Should().Be(VexStatus.Affected); + entry.ContributingSources.Should().Contain("nvd"); + } + + [Fact] + public void AddResolved_ShouldAddResolvedEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to"); + + // Act + builder.AddResolved( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + VexStatus.NotAffected, + 0.80, + 0.95, + VexJustification.VulnerableCodeNotPresent); + + var report = builder.Build(); + + // Assert + report.Summary.ResolvedCount.Should().Be(1); + var entry = report.GetSection(DeltaSection.Resolved)[0]; + entry.FromStatus.Should().Be(VexStatus.Affected); + entry.ToStatus.Should().Be(VexStatus.NotAffected); + entry.Justification.Should().Be(VexJustification.VulnerableCodeNotPresent); + } + + [Fact] + public void AddConfidenceChange_AboveThreshold_ShouldAddEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .WithOptions(new DeltaReportOptions { ConfidenceChangeThreshold = 0.15 }); + + // Act + builder.AddConfidenceChange( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + 0.50, + 0.90); // 40% increase + + var report = builder.Build(); + + // Assert + report.Summary.ConfidenceUpCount.Should().Be(1); + var entry = report.GetSection(DeltaSection.ConfidenceUp)[0]; + entry.FromConfidence.Should().Be(0.50); + entry.ToConfidence.Should().Be(0.90); + } + + [Fact] + public void AddConfidenceChange_BelowThreshold_ShouldNotAddEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .WithOptions(new DeltaReportOptions { ConfidenceChangeThreshold = 0.15 }); + + // Act + builder.AddConfidenceChange( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + 0.80, + 0.85); // Only 5% increase + + var report = builder.Build(); + + // Assert + report.Summary.ConfidenceUpCount.Should().Be(0); + report.Entries.Should().BeEmpty(); + } + + [Fact] + public void AddConfidenceChange_Decrease_ShouldAddConfidenceDownEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .WithOptions(new DeltaReportOptions { ConfidenceChangeThreshold = 0.15 }); + + // Act + builder.AddConfidenceChange( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + 0.90, + 0.50); // 40% decrease + + var report = builder.Build(); + + // Assert + report.Summary.ConfidenceDownCount.Should().Be(1); + report.GetSection(DeltaSection.ConfidenceDown).Should().HaveCount(1); + } + + [Fact] + public void AddPolicyImpact_ShouldAddPolicyImpactEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to"); + + // Act + builder.AddPolicyImpact( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + 0.85, + "Gate decision changed: pass -> fail"); + + var report = builder.Build(); + + // Assert + report.Summary.PolicyImpactCount.Should().Be(1); + report.HasActionableChanges.Should().BeTrue(); + var entry = report.GetSection(DeltaSection.PolicyImpact)[0]; + entry.Summary.Should().Contain("Gate decision changed"); + } + + [Fact] + public void AddDamped_WhenExcluded_ShouldNotAddEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .WithOptions(new DeltaReportOptions { IncludeDamped = false }); + + // Act + builder.AddDamped( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + VexStatus.NotAffected, + 0.80, + 0.75, + "Duration threshold not met"); + + var report = builder.Build(); + + // Assert + report.Summary.DampedCount.Should().Be(0); + } + + [Fact] + public void AddDamped_WhenIncluded_ShouldAddEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .WithOptions(new DeltaReportOptions { IncludeDamped = true }); + + // Act + builder.AddDamped( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + VexStatus.NotAffected, + 0.80, + 0.75, + "Duration threshold not met"); + + var report = builder.Build(); + + // Assert + report.Summary.DampedCount.Should().Be(1); + report.GetSection(DeltaSection.Damped).Should().HaveCount(1); + } + + [Fact] + public void AddEvidenceChange_ShouldAddEvidenceChangedEntry() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .WithOptions(new DeltaReportOptions { IncludeEvidenceChanges = true }); + + // Act + builder.AddEvidenceChange( + "CVE-2024-1234", + "pkg:npm/lodash@4.17.21", + VexStatus.Affected, + 0.85, + "heuristic", + "binary"); + + var report = builder.Build(); + + // Assert + report.Summary.EvidenceChangedCount.Should().Be(1); + var entry = report.GetSection(DeltaSection.EvidenceChanged)[0]; + entry.FromRationaleClass.Should().Be("heuristic"); + entry.ToRationaleClass.Should().Be("binary"); + } + + [Fact] + public void Build_ShouldSortEntriesDeterministically() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to"); + + // Add entries in non-sorted order + builder.AddNew("CVE-2024-0002", "pkg:b", VexStatus.Affected, 0.8); + builder.AddNew("CVE-2024-0001", "pkg:a", VexStatus.Affected, 0.8); + builder.AddResolved("CVE-2024-0003", "pkg:c", VexStatus.Affected, VexStatus.Fixed, 0.8, 0.9); + + // Act + var report = builder.Build(); + + // Assert - entries should be sorted by section then vuln ID then product key + report.Entries[0].Section.Should().Be(DeltaSection.New); + report.Entries[0].VulnerabilityId.Should().Be("CVE-2024-0001"); + report.Entries[1].VulnerabilityId.Should().Be("CVE-2024-0002"); + report.Entries[2].Section.Should().Be(DeltaSection.Resolved); + } + + [Fact] + public void Build_ReportId_ShouldBeDeterministic() + { + // Arrange + var builder1 = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .AddNew("CVE-2024-1234", "pkg:npm/lodash", VexStatus.Affected, 0.8); + + var builder2 = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .AddNew("CVE-2024-1234", "pkg:npm/lodash", VexStatus.Affected, 0.8); + + // Act + var report1 = builder1.Build(); + var report2 = builder2.Build(); + + // Assert - same inputs should produce same report ID + report1.ReportId.Should().Be(report2.ReportId); + } + + [Fact] + public void ToNotificationSummary_WithMultipleChanges_ShouldFormatCorrectly() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .AddNew("CVE-2024-0001", "pkg:a", VexStatus.Affected, 0.8) + .AddNew("CVE-2024-0002", "pkg:b", VexStatus.Affected, 0.8) + .AddResolved("CVE-2024-0003", "pkg:c", VexStatus.Affected, VexStatus.Fixed, 0.8, 0.9); + + // Act + var report = builder.Build(); + var summary = report.ToNotificationSummary(); + + // Assert + summary.Should().Contain("2 new"); + summary.Should().Contain("1 resolved"); + } + + [Fact] + public void ToNotificationSummary_NoChanges_ShouldReturnNoSignificantChanges() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to"); + + // Act + var report = builder.Build(); + var summary = report.ToNotificationSummary(); + + // Assert + summary.Should().Be("No significant changes"); + } + + [Fact] + public void BySection_ShouldGroupEntriesCorrectly() + { + // Arrange + var builder = new DeltaReportBuilder(_timeProvider) + .WithSnapshots("sha256:from", "sha256:to") + .AddNew("CVE-2024-0001", "pkg:a", VexStatus.Affected, 0.8) + .AddResolved("CVE-2024-0002", "pkg:b", VexStatus.Affected, VexStatus.Fixed, 0.8, 0.9); + + // Act + var report = builder.Build(); + var bySection = report.BySection; + + // Assert + bySection.Should().ContainKey(DeltaSection.New); + bySection.Should().ContainKey(DeltaSection.Resolved); + bySection[DeltaSection.New].Should().HaveCount(1); + bySection[DeltaSection.Resolved].Should().HaveCount(1); + } +} diff --git a/src/VexLens/__Tests/StellaOps.VexLens.Tests/NoiseGate/NoiseGateServiceTests.cs b/src/VexLens/__Tests/StellaOps.VexLens.Tests/NoiseGate/NoiseGateServiceTests.cs new file mode 100644 index 000000000..990825a60 --- /dev/null +++ b/src/VexLens/__Tests/StellaOps.VexLens.Tests/NoiseGate/NoiseGateServiceTests.cs @@ -0,0 +1,451 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using NSubstitute; +using StellaOps.Policy.Engine.Gates; +using StellaOps.ReachGraph.Deduplication; +using StellaOps.ReachGraph.Schema; +using StellaOps.VexLens.Models; +using StellaOps.VexLens.NoiseGate; +using Xunit; + +namespace StellaOps.VexLens.Tests.NoiseGate; + +/// +/// Unit tests for . +/// +[Trait("Category", "Unit")] +public class NoiseGateServiceTests +{ + private readonly FakeTimeProvider _timeProvider; + private readonly NoiseGateOptions _defaultOptions; + + public NoiseGateServiceTests() + { + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 4, 12, 0, 0, TimeSpan.Zero)); + _defaultOptions = new NoiseGateOptions + { + Enabled = true, + EdgeDeduplicationEnabled = true, + StabilityDampingEnabled = true, + MinConfidenceThreshold = 0.0, + ConfidenceChangeThreshold = 0.15 + }; + } + + private NoiseGateService CreateService( + IEdgeDeduplicator? edgeDeduplicator = null, + IStabilityDampingGate? dampingGate = null, + NoiseGateOptions? options = null) + { + var opts = options ?? _defaultOptions; + var optionsMonitor = new TestOptionsMonitor(opts); + + edgeDeduplicator ??= new EdgeDeduplicator(); + + if (dampingGate is null) + { + var dampingOptions = new StabilityDampingOptions { Enabled = true }; + var dampingOptionsMonitor = new TestOptionsMonitor(dampingOptions); + dampingGate = new StabilityDampingGate( + dampingOptionsMonitor, + _timeProvider, + NullLogger.Instance); + } + + return new NoiseGateService( + edgeDeduplicator, + dampingGate, + optionsMonitor, + _timeProvider, + NullLogger.Instance); + } + + [Fact] + public async Task DedupeEdgesAsync_WithDuplicateEdges_ShouldDeduplicates() + { + // Arrange + var service = CreateService(); + var edges = new List + { + new() + { + From = "node-a", + To = "node-b", + Why = new EdgeExplanation { Type = EdgeExplanationType.DirectCall, Confidence = 0.9, Loc = "file1.cs:10" } + }, + new() + { + From = "node-a", + To = "node-b", + Why = new EdgeExplanation { Type = EdgeExplanationType.DirectCall, Confidence = 0.85, Loc = "file2.cs:20" } + } + }; + + // Act + var result = await service.DedupeEdgesAsync(edges); + + // Assert + result.Should().HaveCount(1); + result[0].EntryPointId.Should().Be("node-a"); + result[0].SinkId.Should().Be("node-b"); + result[0].ProvenanceCount.Should().Be(2); + } + + [Fact] + public async Task DedupeEdgesAsync_WhenDisabled_ShouldPassThrough() + { + // Arrange + var options = new NoiseGateOptions { Enabled = false }; + var service = CreateService(options: options); + var edges = new List + { + new() + { + From = "node-a", + To = "node-b", + Why = new EdgeExplanation { Type = EdgeExplanationType.DirectCall, Confidence = 0.9 } + }, + new() + { + From = "node-a", + To = "node-b", + Why = new EdgeExplanation { Type = EdgeExplanationType.DirectCall, Confidence = 0.85 } + } + }; + + // Act + var result = await service.DedupeEdgesAsync(edges); + + // Assert + result.Should().HaveCount(2); // Not deduplicated + } + + [Fact] + public async Task ResolveVerdictAsync_NewVerdict_ShouldSurface() + { + // Arrange + var service = CreateService(); + var request = new VerdictResolutionRequest + { + Key = "artifact:CVE-2024-1234", + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + ProposedStatus = VexStatus.Affected, + ProposedConfidence = 0.85 + }; + + // Act + var result = await service.ResolveVerdictAsync(request); + + // Assert + result.WasSurfaced.Should().BeTrue(); + result.VulnerabilityId.Should().Be("CVE-2024-1234"); + result.Status.Should().Be(VexStatus.Affected); + } + + [Fact] + public async Task ResolveVerdictAsync_WhenDampingDisabled_ShouldAlwaysSurface() + { + // Arrange + var options = new NoiseGateOptions { StabilityDampingEnabled = false }; + var service = CreateService(options: options); + var request = new VerdictResolutionRequest + { + Key = "artifact:CVE-2024-1234", + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + ProposedStatus = VexStatus.Affected, + ProposedConfidence = 0.85 + }; + + // Act + var result = await service.ResolveVerdictAsync(request); + + // Assert + result.WasSurfaced.Should().BeTrue(); + result.DampingReason.Should().BeNull(); + } + + [Fact] + public async Task GateAsync_ShouldDeduplicateAndResolve() + { + // Arrange + var service = CreateService(); + var graph = new ReachGraphMinimal + { + SchemaVersion = "reachgraph.min@v1", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Scope = new ReachGraphScope(["main"], ["*"]), + Nodes = [new ReachGraphNode { Id = "node-a" }, new ReachGraphNode { Id = "node-b" }], + Edges = + [ + new ReachGraphEdge + { + From = "node-a", + To = "node-b", + Why = new EdgeExplanation { Type = EdgeExplanationType.DirectCall, Confidence = 0.9 } + }, + new ReachGraphEdge + { + From = "node-a", + To = "node-b", + Why = new EdgeExplanation { Type = EdgeExplanationType.DirectCall, Confidence = 0.85 } + } + ], + Provenance = new ReachGraphProvenance("scanner", "1.0", _timeProvider.GetUtcNow()) + }; + + var request = new NoiseGateRequest + { + Graph = graph, + SnapshotId = "snapshot-001", + Verdicts = + [ + new VerdictResolutionRequest + { + Key = "artifact:CVE-2024-1234", + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + ProposedStatus = VexStatus.Affected, + ProposedConfidence = 0.85 + } + ] + }; + + // Act + var result = await service.GateAsync(request); + + // Assert + result.SnapshotId.Should().Be("snapshot-001"); + result.Edges.Should().HaveCount(1); // Deduplicated + result.Verdicts.Should().HaveCount(1); + result.Statistics.EdgeReductionPercent.Should().Be(50.0); + } + + [Fact] + public async Task DiffAsync_ShouldDetectNewFindings() + { + // Arrange + var service = CreateService(); + + var fromSnapshot = new GatedGraphSnapshot + { + SnapshotId = "snapshot-001", + Digest = "sha256:from", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Edges = [], + Verdicts = [], + CreatedAt = _timeProvider.GetUtcNow(), + Statistics = new GatingStatistics + { + OriginalEdgeCount = 0, + DeduplicatedEdgeCount = 0, + TotalVerdictCount = 0, + SurfacedVerdictCount = 0, + DampedVerdictCount = 0, + Duration = TimeSpan.Zero + } + }; + + var toSnapshot = new GatedGraphSnapshot + { + SnapshotId = "snapshot-002", + Digest = "sha256:to", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Edges = [], + Verdicts = + [ + new ResolvedVerdict + { + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + Status = VexStatus.Affected, + Confidence = 0.85, + WasSurfaced = true, + ResolvedAt = _timeProvider.GetUtcNow() + } + ], + CreatedAt = _timeProvider.GetUtcNow(), + Statistics = new GatingStatistics + { + OriginalEdgeCount = 0, + DeduplicatedEdgeCount = 0, + TotalVerdictCount = 1, + SurfacedVerdictCount = 1, + DampedVerdictCount = 0, + Duration = TimeSpan.Zero + } + }; + + // Act + var delta = await service.DiffAsync(fromSnapshot, toSnapshot); + + // Assert + delta.Summary.NewCount.Should().Be(1); + delta.Summary.ResolvedCount.Should().Be(0); + delta.Entries.Should().ContainSingle(e => e.Section == Delta.DeltaSection.New); + } + + [Fact] + public async Task DiffAsync_ShouldDetectResolvedFindings() + { + // Arrange + var service = CreateService(); + + var fromSnapshot = new GatedGraphSnapshot + { + SnapshotId = "snapshot-001", + Digest = "sha256:from", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Edges = [], + Verdicts = + [ + new ResolvedVerdict + { + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + Status = VexStatus.Affected, + Confidence = 0.85, + WasSurfaced = true, + ResolvedAt = _timeProvider.GetUtcNow() + } + ], + CreatedAt = _timeProvider.GetUtcNow(), + Statistics = new GatingStatistics + { + OriginalEdgeCount = 0, + DeduplicatedEdgeCount = 0, + TotalVerdictCount = 1, + SurfacedVerdictCount = 1, + DampedVerdictCount = 0, + Duration = TimeSpan.Zero + } + }; + + var toSnapshot = new GatedGraphSnapshot + { + SnapshotId = "snapshot-002", + Digest = "sha256:to", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Edges = [], + Verdicts = + [ + new ResolvedVerdict + { + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + Status = VexStatus.NotAffected, + Confidence = 0.95, + WasSurfaced = true, + Justification = VexJustification.VulnerableCodeNotPresent, + ResolvedAt = _timeProvider.GetUtcNow() + } + ], + CreatedAt = _timeProvider.GetUtcNow(), + Statistics = new GatingStatistics + { + OriginalEdgeCount = 0, + DeduplicatedEdgeCount = 0, + TotalVerdictCount = 1, + SurfacedVerdictCount = 1, + DampedVerdictCount = 0, + Duration = TimeSpan.Zero + } + }; + + // Act + var delta = await service.DiffAsync(fromSnapshot, toSnapshot); + + // Assert + delta.Summary.ResolvedCount.Should().Be(1); + delta.Entries.Should().ContainSingle(e => e.Section == Delta.DeltaSection.Resolved); + } + + [Fact] + public async Task DiffAsync_ShouldDetectConfidenceChanges() + { + // Arrange + var service = CreateService(); + + var fromSnapshot = new GatedGraphSnapshot + { + SnapshotId = "snapshot-001", + Digest = "sha256:from", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Edges = [], + Verdicts = + [ + new ResolvedVerdict + { + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + Status = VexStatus.Affected, + Confidence = 0.50, + WasSurfaced = true, + ResolvedAt = _timeProvider.GetUtcNow() + } + ], + CreatedAt = _timeProvider.GetUtcNow(), + Statistics = new GatingStatistics + { + OriginalEdgeCount = 0, + DeduplicatedEdgeCount = 0, + TotalVerdictCount = 1, + SurfacedVerdictCount = 1, + DampedVerdictCount = 0, + Duration = TimeSpan.Zero + } + }; + + var toSnapshot = new GatedGraphSnapshot + { + SnapshotId = "snapshot-002", + Digest = "sha256:to", + Artifact = new ReachGraphArtifact("test", "sha256:abc123", []), + Edges = [], + Verdicts = + [ + new ResolvedVerdict + { + VulnerabilityId = "CVE-2024-1234", + ProductKey = "pkg:npm/lodash@4.17.21", + Status = VexStatus.Affected, + Confidence = 0.90, // Large increase + WasSurfaced = true, + ResolvedAt = _timeProvider.GetUtcNow() + } + ], + CreatedAt = _timeProvider.GetUtcNow(), + Statistics = new GatingStatistics + { + OriginalEdgeCount = 0, + DeduplicatedEdgeCount = 0, + TotalVerdictCount = 1, + SurfacedVerdictCount = 1, + DampedVerdictCount = 0, + Duration = TimeSpan.Zero + } + }; + + // Act + var delta = await service.DiffAsync(fromSnapshot, toSnapshot); + + // Assert + delta.Summary.ConfidenceUpCount.Should().Be(1); + delta.Entries.Should().ContainSingle(e => e.Section == Delta.DeltaSection.ConfidenceUp); + } + + private sealed class TestOptionsMonitor : IOptionsMonitor + where T : class + { + public TestOptionsMonitor(T value) => CurrentValue = value; + public T CurrentValue { get; } + public T Get(string? name) => CurrentValue; + public IDisposable? OnChange(Action listener) => null; + } +} diff --git a/src/VexLens/__Tests/StellaOps.VexLens.Tests/StellaOps.VexLens.Tests.csproj b/src/VexLens/__Tests/StellaOps.VexLens.Tests/StellaOps.VexLens.Tests.csproj new file mode 100644 index 000000000..1fb95a206 --- /dev/null +++ b/src/VexLens/__Tests/StellaOps.VexLens.Tests/StellaOps.VexLens.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + preview + enable + enable + false + false + true + StellaOps.VexLens.Tests + + + + + + + + + + + + + + diff --git a/src/Web/StellaOps.Web/src/app/core/api/noise-gating.client.ts b/src/Web/StellaOps.Web/src/app/core/api/noise-gating.client.ts new file mode 100644 index 000000000..5756342d3 --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/core/api/noise-gating.client.ts @@ -0,0 +1,302 @@ +/** + * Noise-Gating API client. + * Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui (NG-FE-003) + * Description: API client for noise-gating delta reports from VexLens. + */ + +import { HttpClient, HttpHeaders, HttpParams } from '@angular/common/http'; +import { Injectable, InjectionToken, inject, signal, computed } from '@angular/core'; +import { Observable, throwError, of, shareReplay } from 'rxjs'; +import { map, catchError, tap } from 'rxjs/operators'; + +import { AuthSessionStore } from '../auth/auth-session.store'; +import { generateTraceId } from './trace.util'; +import { + NoiseGatingDeltaReport, + ComputeDeltaRequest, + GatedSnapshotResponse, + GateSnapshotRequest, + AggregatedGatingStatistics, + GatingStatisticsQuery, +} from './noise-gating.models'; + +/** + * Query options for noise-gating API calls. + */ +export interface NoiseGatingQueryOptions { + traceId?: string; + bypassCache?: boolean; +} + +/** + * Noise-gating API interface. + */ +export interface NoiseGatingApi { + // Delta operations + computeDelta(request: ComputeDeltaRequest, options?: NoiseGatingQueryOptions): Observable; + + // Snapshot operations + gateSnapshot(snapshotId: string, request: GateSnapshotRequest, options?: NoiseGatingQueryOptions): Observable; + + // Statistics + getGatingStatistics(query?: GatingStatisticsQuery, options?: NoiseGatingQueryOptions): Observable; +} + +export const NOISE_GATING_API = new InjectionToken('NOISE_GATING_API'); +export const NOISE_GATING_API_BASE_URL = new InjectionToken('NOISE_GATING_API_BASE_URL'); + +const normalizeBaseUrl = (baseUrl: string): string => + baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl; + +/** + * HTTP implementation of noise-gating API client. + */ +@Injectable({ providedIn: 'root' }) +export class NoiseGatingApiHttpClient implements NoiseGatingApi { + private readonly http = inject(HttpClient); + private readonly authSession = inject(AuthSessionStore); + private readonly baseUrl = normalizeBaseUrl( + inject(NOISE_GATING_API_BASE_URL, { optional: true }) ?? '/api/v1/vexlens' + ); + + // Cache for delta reports (key: fromId|toId) + private readonly deltaCache = new Map>(); + + // Signal-based state for current delta report + private readonly _currentReport = signal(null); + private readonly _loading = signal(false); + private readonly _error = signal(null); + + /** Current delta report signal */ + readonly currentReport = this._currentReport.asReadonly(); + + /** Loading state signal */ + readonly loading = this._loading.asReadonly(); + + /** Error state signal */ + readonly error = this._error.asReadonly(); + + /** Computed: whether current report has actionable changes */ + readonly hasActionableChanges = computed(() => this._currentReport()?.hasActionableChanges ?? false); + + /** Computed: summary from current report */ + readonly summary = computed(() => this._currentReport()?.summary ?? null); + + computeDelta( + request: ComputeDeltaRequest, + options: NoiseGatingQueryOptions = {} + ): Observable { + const traceId = options.traceId ?? generateTraceId(); + const headers = this.buildHeaders(traceId); + const cacheKey = `${request.fromSnapshotId}|${request.toSnapshotId}`; + + // Check cache unless bypass requested + if (!options.bypassCache && this.deltaCache.has(cacheKey)) { + return this.deltaCache.get(cacheKey)!; + } + + this._loading.set(true); + this._error.set(null); + + const obs = this.http + .post(`${this.baseUrl}/deltas/compute`, request, { headers }) + .pipe( + tap((report) => { + this._currentReport.set(report); + this._loading.set(false); + }), + catchError((err) => { + this._loading.set(false); + this._error.set(this.extractErrorMessage(err, traceId)); + return throwError(() => this.mapError(err, traceId)); + }), + shareReplay(1) + ); + + this.deltaCache.set(cacheKey, obs); + return obs; + } + + gateSnapshot( + snapshotId: string, + request: GateSnapshotRequest, + options: NoiseGatingQueryOptions = {} + ): Observable { + const traceId = options.traceId ?? generateTraceId(); + const headers = this.buildHeaders(traceId); + + return this.http + .post( + `${this.baseUrl}/gating/snapshots/${encodeURIComponent(snapshotId)}/gate`, + request, + { headers } + ) + .pipe(catchError((err) => throwError(() => this.mapError(err, traceId)))); + } + + getGatingStatistics( + query: GatingStatisticsQuery = {}, + options: NoiseGatingQueryOptions = {} + ): Observable { + const traceId = options.traceId ?? generateTraceId(); + const headers = this.buildHeaders(traceId); + let params = new HttpParams(); + + if (query.tenantId) params = params.set('tenantId', query.tenantId); + if (query.fromDate) params = params.set('fromDate', query.fromDate); + if (query.toDate) params = params.set('toDate', query.toDate); + + return this.http + .get(`${this.baseUrl}/gating/statistics`, { headers, params }) + .pipe(catchError((err) => throwError(() => this.mapError(err, traceId)))); + } + + /** Clear the delta cache */ + clearCache(): void { + this.deltaCache.clear(); + this._currentReport.set(null); + this._error.set(null); + } + + /** Clear current report state */ + clearCurrentReport(): void { + this._currentReport.set(null); + this._error.set(null); + } + + private buildHeaders(traceId: string): HttpHeaders { + const tenant = this.authSession.getActiveTenantId() || ''; + return new HttpHeaders({ + 'X-StellaOps-Tenant': tenant, + 'X-Stella-Trace-Id': traceId, + 'X-Stella-Request-Id': traceId, + 'Content-Type': 'application/json', + Accept: 'application/json', + }); + } + + private extractErrorMessage(err: unknown, traceId: string): string { + if (err && typeof err === 'object' && 'error' in err) { + const httpError = err as { error?: { message?: string } }; + if (httpError.error?.message) { + return httpError.error.message; + } + } + if (err instanceof Error) { + return err.message; + } + return `Request failed (trace: ${traceId})`; + } + + private mapError(err: unknown, traceId: string): Error { + if (err instanceof Error) { + return new Error(`[${traceId}] Noise-gating error: ${err.message}`); + } + return new Error(`[${traceId}] Noise-gating error: Unknown error`); + } +} + +/** + * Mock implementation for testing. + */ +@Injectable({ providedIn: 'root' }) +export class MockNoiseGatingClient implements NoiseGatingApi { + private readonly mockReport: NoiseGatingDeltaReport = { + reportId: 'delta-mock-001', + fromSnapshotDigest: 'sha256:abc123', + toSnapshotDigest: 'sha256:def456', + generatedAt: new Date().toISOString(), + entries: [ + { + section: 'new', + vulnerabilityId: 'CVE-2024-12345', + productKey: 'pkg:npm/lodash@4.17.20', + toStatus: 'affected', + toConfidence: 0.85, + summary: 'New affected finding', + createdAt: new Date().toISOString(), + }, + { + section: 'resolved', + vulnerabilityId: 'CVE-2024-11111', + productKey: 'pkg:npm/axios@1.6.0', + fromStatus: 'affected', + toStatus: 'not_affected', + fromConfidence: 0.75, + toConfidence: 0.95, + justification: 'vulnerable_code_not_present', + summary: 'Resolved: affected -> not_affected', + createdAt: new Date().toISOString(), + }, + { + section: 'confidence_up', + vulnerabilityId: 'CVE-2024-22222', + productKey: 'pkg:npm/express@4.18.2', + fromStatus: 'affected', + toStatus: 'affected', + fromConfidence: 0.6, + toConfidence: 0.9, + summary: 'Confidence increased: 60% -> 90%', + createdAt: new Date().toISOString(), + }, + ], + summary: { + totalCount: 3, + newCount: 1, + resolvedCount: 1, + confidenceUpCount: 1, + confidenceDownCount: 0, + policyImpactCount: 0, + dampedCount: 0, + evidenceChangedCount: 0, + }, + hasActionableChanges: true, + }; + + computeDelta( + _request: ComputeDeltaRequest, + _options?: NoiseGatingQueryOptions + ): Observable { + return of(this.mockReport); + } + + gateSnapshot( + snapshotId: string, + _request: GateSnapshotRequest, + _options?: NoiseGatingQueryOptions + ): Observable { + return of({ + snapshotId, + digest: 'sha256:mock123', + createdAt: new Date().toISOString(), + edgeCount: 150, + verdictCount: 45, + statistics: { + originalEdgeCount: 200, + deduplicatedEdgeCount: 150, + edgeReductionPercent: 25, + totalVerdictCount: 50, + surfacedVerdictCount: 45, + dampedVerdictCount: 5, + duration: '00:00:01.234', + }, + }); + } + + getGatingStatistics( + _query?: GatingStatisticsQuery, + _options?: NoiseGatingQueryOptions + ): Observable { + return of({ + totalSnapshots: 100, + totalEdgesProcessed: 15000, + totalEdgesAfterDedup: 12000, + averageEdgeReductionPercent: 20, + totalVerdicts: 5000, + totalSurfaced: 4500, + totalDamped: 500, + averageDampingPercent: 10, + computedAt: new Date().toISOString(), + }); + } +} diff --git a/src/Web/StellaOps.Web/src/app/core/api/noise-gating.models.ts b/src/Web/StellaOps.Web/src/app/core/api/noise-gating.models.ts new file mode 100644 index 000000000..02c59d62d --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/core/api/noise-gating.models.ts @@ -0,0 +1,266 @@ +/** + * Noise-Gating Delta Report Models + * Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui + * Description: TypeScript models for noise-gating delta reports from VexLens. + */ + +import { VexStatementStatus } from './vex-hub.models'; + +// Delta section types - matches backend DeltaSection enum +export type NoiseGatingDeltaSection = + | 'new' + | 'resolved' + | 'confidence_up' + | 'confidence_down' + | 'policy_impact' + | 'damped' + | 'evidence_changed'; + +// VEX justification types for delta entries +export type VexJustification = + | 'component_not_present' + | 'vulnerable_code_not_present' + | 'vulnerable_code_not_in_execute_path' + | 'vulnerable_code_cannot_be_controlled_by_adversary' + | 'inline_mitigations_already_exist'; + +/** + * Single delta entry in API format. + */ +export interface NoiseGatingDeltaEntry { + readonly section: NoiseGatingDeltaSection; + readonly vulnerabilityId: string; + readonly productKey: string; + readonly fromStatus?: VexStatementStatus; + readonly toStatus?: VexStatementStatus; + readonly fromConfidence?: number; + readonly toConfidence?: number; + readonly justification?: VexJustification; + readonly fromRationaleClass?: string; + readonly toRationaleClass?: string; + readonly summary?: string; + readonly contributingSources?: readonly string[]; + readonly createdAt: string; +} + +/** + * Summary counts for delta report. + */ +export interface NoiseGatingDeltaSummary { + readonly totalCount: number; + readonly newCount: number; + readonly resolvedCount: number; + readonly confidenceUpCount: number; + readonly confidenceDownCount: number; + readonly policyImpactCount: number; + readonly dampedCount: number; + readonly evidenceChangedCount: number; +} + +/** + * Delta report response from backend. + */ +export interface NoiseGatingDeltaReport { + readonly reportId: string; + readonly fromSnapshotDigest: string; + readonly toSnapshotDigest: string; + readonly generatedAt: string; + readonly entries: readonly NoiseGatingDeltaEntry[]; + readonly summary: NoiseGatingDeltaSummary; + readonly hasActionableChanges: boolean; +} + +/** + * Request to compute delta between two snapshots. + */ +export interface ComputeDeltaRequest { + readonly fromSnapshotId: string; + readonly toSnapshotId: string; + readonly tenantId?: string; + readonly options?: DeltaReportOptionsRequest; +} + +/** + * Options for delta report computation. + */ +export interface DeltaReportOptionsRequest { + readonly confidenceChangeThreshold?: number; + readonly includeDamped?: boolean; + readonly includeEvidenceChanges?: boolean; +} + +/** + * Gating statistics for API response. + */ +export interface GatingStatistics { + readonly originalEdgeCount: number; + readonly deduplicatedEdgeCount: number; + readonly edgeReductionPercent: number; + readonly totalVerdictCount: number; + readonly surfacedVerdictCount: number; + readonly dampedVerdictCount: number; + readonly duration: string; +} + +/** + * Response from gating a snapshot. + */ +export interface GatedSnapshotResponse { + readonly snapshotId: string; + readonly digest: string; + readonly createdAt: string; + readonly edgeCount: number; + readonly verdictCount: number; + readonly statistics: GatingStatistics; +} + +/** + * Request to gate a graph snapshot. + */ +export interface GateSnapshotRequest { + readonly snapshotId: string; + readonly tenantId?: string; + readonly options?: NoiseGateOptionsRequest; +} + +/** + * Options for noise-gating. + */ +export interface NoiseGateOptionsRequest { + readonly edgeDeduplicationEnabled?: boolean; + readonly stabilityDampingEnabled?: boolean; + readonly minConfidenceThreshold?: number; + readonly confidenceChangeThreshold?: number; +} + +/** + * Aggregated gating statistics response. + */ +export interface AggregatedGatingStatistics { + readonly totalSnapshots: number; + readonly totalEdgesProcessed: number; + readonly totalEdgesAfterDedup: number; + readonly averageEdgeReductionPercent: number; + readonly totalVerdicts: number; + readonly totalSurfaced: number; + readonly totalDamped: number; + readonly averageDampingPercent: number; + readonly computedAt: string; +} + +/** + * Query parameters for gating statistics. + */ +export interface GatingStatisticsQuery { + readonly tenantId?: string; + readonly fromDate?: string; + readonly toDate?: string; +} + +// === Helper Functions === + +/** + * Get display label for delta section. + */ +export function getSectionLabel(section: NoiseGatingDeltaSection): string { + switch (section) { + case 'new': return 'New'; + case 'resolved': return 'Resolved'; + case 'confidence_up': return 'Confidence Up'; + case 'confidence_down': return 'Confidence Down'; + case 'policy_impact': return 'Policy Impact'; + case 'damped': return 'Damped'; + case 'evidence_changed': return 'Evidence Changed'; + default: return section; + } +} + +/** + * Get CSS color class for delta section. + */ +export function getSectionColorClass(section: NoiseGatingDeltaSection): string { + switch (section) { + case 'new': return 'section-new'; + case 'resolved': return 'section-resolved'; + case 'confidence_up': return 'section-confidence-up'; + case 'confidence_down': return 'section-confidence-down'; + case 'policy_impact': return 'section-policy-impact'; + case 'damped': return 'section-damped'; + case 'evidence_changed': return 'section-evidence'; + default: return 'section-unknown'; + } +} + +/** + * Get icon for delta section (ASCII-only per CLAUDE.md rules). + */ +export function getSectionIcon(section: NoiseGatingDeltaSection): string { + switch (section) { + case 'new': return '+'; + case 'resolved': return '-'; + case 'confidence_up': return '^'; + case 'confidence_down': return 'v'; + case 'policy_impact': return '!'; + case 'damped': return '~'; + case 'evidence_changed': return '*'; + default: return '?'; + } +} + +/** + * Format confidence as percentage. + */ +export function formatConfidence(confidence?: number): string { + if (confidence === undefined || confidence === null) return '--'; + return (confidence * 100).toFixed(0) + '%'; +} + +/** + * Format confidence delta. + */ +export function formatConfidenceDelta(from?: number, to?: number): string { + if (from === undefined || to === undefined) return '--'; + const delta = (to - from) * 100; + const sign = delta >= 0 ? '+' : ''; + return sign + delta.toFixed(0) + '%'; +} + +/** + * Check if a section represents an actionable change. + */ +export function isActionableSection(section: NoiseGatingDeltaSection): boolean { + return section === 'new' || section === 'policy_impact' || section === 'confidence_up'; +} + +/** + * Get the priority order for section sorting. + */ +export function getSectionPriority(section: NoiseGatingDeltaSection): number { + switch (section) { + case 'new': return 1; + case 'policy_impact': return 2; + case 'confidence_up': return 3; + case 'confidence_down': return 4; + case 'resolved': return 5; + case 'evidence_changed': return 6; + case 'damped': return 7; + default: return 99; + } +} + +/** + * Group entries by section. + */ +export function groupEntriesBySection( + entries: readonly NoiseGatingDeltaEntry[] +): Map { + const grouped = new Map(); + + for (const entry of entries) { + const existing = grouped.get(entry.section) ?? []; + existing.push(entry); + grouped.set(entry.section, existing); + } + + return grouped; +} diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/index.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/index.ts index 1d2df15d6..d67e78770 100644 --- a/src/Web/StellaOps.Web/src/app/features/triage/components/index.ts +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/index.ts @@ -58,3 +58,11 @@ export { VexTrustDisplayComponent } from './vex-trust-display/vex-trust-display. export { ReplayCommandComponent } from './replay-command/replay-command.component'; export { VerdictLadderComponent } from './verdict-ladder/verdict-ladder.component'; export { CaseHeaderComponent } from './case-header/case-header.component'; + +// Noise-Gating Delta Report (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) +export { + NoiseGatingSummaryStripComponent, + DeltaEntryCardComponent, + NoiseGatingDeltaReportComponent, + GatingStatisticsCardComponent, +} from './noise-gating'; diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/delta-entry-card.component.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/delta-entry-card.component.ts new file mode 100644 index 000000000..032fbe75a --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/delta-entry-card.component.ts @@ -0,0 +1,344 @@ +// ----------------------------------------------------------------------------- +// delta-entry-card.component.ts +// Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui +// Task: NG-FE-005 - DeltaEntryCardComponent +// Description: Card component for displaying individual delta entries. +// Shows CVE ID, package, status transition, and confidence changes. +// ----------------------------------------------------------------------------- + +import { Component, input, output, computed } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { + NoiseGatingDeltaEntry, + getSectionLabel, + getSectionColorClass, + formatConfidence, + formatConfidenceDelta, + isActionableSection, +} from '../../../../core/api/noise-gating.models'; + +/** + * Card component for individual delta entries. + */ +@Component({ + selector: 'app-delta-entry-card', + standalone: true, + imports: [CommonModule], + template: ` +
+ +
+ + {{ sectionLabel() }} + +
+ + +
+ +
+ {{ entry().vulnerabilityId }} +
+ + +
+ {{ shortProductKey() }} +
+ + + @if (hasStatusChange()) { +
+ + {{ entry().fromStatus ?? '-' }} + + -> + + {{ entry().toStatus ?? '-' }} + +
+ } + + + @if (hasConfidenceChange()) { +
+ {{ fromConfidence() }} + -> + {{ toConfidence() }} + + ({{ confidenceDelta() }}) + +
+ } + + + @if (entry().summary) { +
{{ entry().summary }}
+ } + + + @if (entry().justification) { +
+ Justification: + {{ formatJustification(entry().justification) }} +
+ } + + + @if (entry().contributingSources?.length) { +
+ Sources: + @for (source of entry().contributingSources; track source) { + {{ source }} + } +
+ } +
+ + +
+ {{ formattedTimestamp() }} +
+
+ `, + styles: [` + .delta-entry-card { + display: flex; + flex-direction: column; + gap: 0.5rem; + padding: 0.875rem 1rem; + background: var(--bg-primary, #fff); + border: 1px solid var(--border-color, #e5e7eb); + border-radius: 8px; + cursor: pointer; + transition: border-color 0.15s, box-shadow 0.15s, transform 0.15s; + } + .delta-entry-card:hover { + border-color: var(--primary-color, #3b82f6); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); + } + .delta-entry-card:focus { + outline: 2px solid var(--primary-color, #3b82f6); + outline-offset: 2px; + } + .delta-entry-card.actionable { + border-left: 3px solid var(--warning-color, #f59e0b); + } + + /* Section-specific left border colors */ + .delta-entry-card.section-new { border-left-color: #22c55e; } + .delta-entry-card.section-resolved { border-left-color: #3b82f6; } + .delta-entry-card.section-confidence-up { border-left-color: #14b8a6; } + .delta-entry-card.section-confidence-down { border-left-color: #f97316; } + .delta-entry-card.section-policy-impact { border-left-color: #ef4444; } + .delta-entry-card.section-damped { border-left-color: #9ca3af; } + .delta-entry-card.section-evidence { border-left-color: #8b5cf6; } + + .delta-entry-card__section { + display: flex; + justify-content: flex-start; + } + .delta-entry-card__section-badge { + display: inline-block; + padding: 0.125rem 0.5rem; + background: var(--bg-secondary, #f3f4f6); + border-radius: 4px; + font-size: 0.6875rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--text-secondary, #6b7280); + } + + .delta-entry-card__content { + display: flex; + flex-direction: column; + gap: 0.375rem; + } + + .delta-entry-card__vuln-id { + font-weight: 600; + font-size: 0.9375rem; + color: var(--text-primary, #111827); + font-family: var(--font-mono, monospace); + } + + .delta-entry-card__product { + font-size: 0.8125rem; + color: var(--text-secondary, #6b7280); + font-family: var(--font-mono, monospace); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .delta-entry-card__status-change, + .delta-entry-card__confidence { + display: flex; + align-items: center; + gap: 0.375rem; + font-size: 0.8125rem; + } + .delta-entry-card__arrow { + color: var(--text-muted, #9ca3af); + font-family: monospace; + } + .delta-entry-card__status { + padding: 0.125rem 0.375rem; + border-radius: 4px; + font-size: 0.75rem; + font-weight: 500; + } + .delta-entry-card__status--from { + background: #f3f4f6; + color: #6b7280; + } + .delta-entry-card__status--to { + background: #dbeafe; + color: #1e40af; + } + + .delta-entry-card__confidence-from, + .delta-entry-card__confidence-to { + font-family: var(--font-mono, monospace); + font-weight: 500; + } + .delta-entry-card__confidence-delta { + font-size: 0.75rem; + font-weight: 600; + } + .delta-entry-card__confidence-delta.positive { color: #15803d; } + .delta-entry-card__confidence-delta.negative { color: #dc2626; } + + .delta-entry-card__summary { + font-size: 0.8125rem; + color: var(--text-secondary, #6b7280); + line-height: 1.4; + } + + .delta-entry-card__justification { + font-size: 0.75rem; + color: var(--text-muted, #9ca3af); + } + .delta-entry-card__justification-label { + font-weight: 500; + margin-right: 0.25rem; + } + + .delta-entry-card__sources { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.25rem; + font-size: 0.75rem; + } + .delta-entry-card__sources-label { + color: var(--text-muted, #9ca3af); + font-weight: 500; + } + .delta-entry-card__source-tag { + padding: 0.125rem 0.375rem; + background: #f3f4f6; + border-radius: 3px; + font-size: 0.6875rem; + color: #6b7280; + } + + .delta-entry-card__timestamp { + font-size: 0.6875rem; + color: var(--text-muted, #9ca3af); + text-align: right; + } + `], +}) +export class DeltaEntryCardComponent { + /** The delta entry to display */ + readonly entry = input.required(); + + /** Emits when card is clicked */ + readonly cardClick = output(); + + /** Section label */ + readonly sectionLabel = computed(() => getSectionLabel(this.entry().section)); + + /** Section CSS class */ + readonly sectionClass = computed(() => getSectionColorClass(this.entry().section)); + + /** Whether this is an actionable section */ + readonly isActionable = computed(() => isActionableSection(this.entry().section)); + + /** Short product key (last part of PURL) */ + readonly shortProductKey = computed(() => { + const purl = this.entry().productKey; + // Extract name@version from PURL like pkg:npm/name@version + const match = purl.match(/\/([^/]+)$/); + return match ? match[1] : purl; + }); + + /** Whether there's a status change */ + readonly hasStatusChange = computed(() => { + const e = this.entry(); + return e.fromStatus !== undefined && e.toStatus !== undefined && e.fromStatus !== e.toStatus; + }); + + /** Whether there's a confidence change */ + readonly hasConfidenceChange = computed(() => { + const e = this.entry(); + return e.fromConfidence !== undefined && e.toConfidence !== undefined; + }); + + /** Formatted from confidence */ + readonly fromConfidence = computed(() => formatConfidence(this.entry().fromConfidence)); + + /** Formatted to confidence */ + readonly toConfidence = computed(() => formatConfidence(this.entry().toConfidence)); + + /** Formatted confidence delta */ + readonly confidenceDelta = computed(() => + formatConfidenceDelta(this.entry().fromConfidence, this.entry().toConfidence) + ); + + /** Whether confidence increased */ + readonly isConfidenceUp = computed(() => { + const e = this.entry(); + if (e.fromConfidence === undefined || e.toConfidence === undefined) return false; + return e.toConfidence > e.fromConfidence; + }); + + /** Formatted timestamp */ + readonly formattedTimestamp = computed(() => { + const ts = this.entry().createdAt; + if (!ts) return ''; + try { + const date = new Date(ts); + return date.toLocaleDateString(undefined, { + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + }); + } catch { + return ts; + } + }); + + /** Handle card click */ + onCardClick(): void { + this.cardClick.emit(this.entry()); + } + + /** Format justification for display */ + formatJustification(j: string | undefined): string { + if (!j) return ''; + // Convert snake_case to Title Case + return j.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()); + } +} diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/gating-statistics-card.component.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/gating-statistics-card.component.ts new file mode 100644 index 000000000..ebe4b0580 --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/gating-statistics-card.component.ts @@ -0,0 +1,364 @@ +// ----------------------------------------------------------------------------- +// gating-statistics-card.component.ts +// Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui +// Task: NG-FE-007 - GatingStatisticsCardComponent +// Description: Card component displaying noise-gating statistics. +// Shows edge reduction and verdict damping metrics. +// ----------------------------------------------------------------------------- + +import { Component, input, computed } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { + GatingStatistics, + AggregatedGatingStatistics, +} from '../../../../core/api/noise-gating.models'; + +/** + * Card component for displaying gating statistics. + */ +@Component({ + selector: 'app-gating-statistics-card', + standalone: true, + imports: [CommonModule], + template: ` +
+
+

{{ title() }}

+ @if (computedAt()) { + {{ formattedComputedAt() }} + } +
+ +
+ +
+

Edge Deduplication

+
+
+ Original + {{ edgesOriginal() }} +
+
->
+
+ After Dedup + {{ edgesDeduped() }} +
+
+
+
+
+
+ + {{ edgeReductionPercent().toFixed(1) }}% reduction + +
+
+ + +
+

Verdict Damping

+
+
+ + + {{ verdictsSurfaced() }} + Surfaced +
+
+ ~ + {{ verdictsDamped() }} + Damped +
+
+ = + {{ verdictsTotal() }} + Total +
+
+ @if (verdictsTotal() > 0) { +
+
+
+
+ } +
+ + + @if (duration()) { +
+ Processing time: + {{ duration() }} +
+ } + + + @if (showAggregated() && aggregatedStats()) { +
+

Aggregated

+
+
+ {{ aggregatedStats()!.totalSnapshots }} + snapshots +
+
+ {{ aggregatedStats()!.averageEdgeReductionPercent.toFixed(1) }}% + avg reduction +
+
+ {{ aggregatedStats()!.averageDampingPercent.toFixed(1) }}% + avg damping +
+
+
+ } +
+
+ `, + styles: [` + .gating-stats-card { + background: var(--bg-primary, #fff); + border: 1px solid var(--border-color, #e5e7eb); + border-radius: 8px; + overflow: hidden; + } + + .gating-stats-card__header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--border-color, #e5e7eb); + background: var(--bg-secondary, #f9fafb); + } + .gating-stats-card__title { + margin: 0; + font-size: 0.875rem; + font-weight: 600; + color: var(--text-primary, #111827); + } + .gating-stats-card__timestamp { + font-size: 0.6875rem; + color: var(--text-muted, #9ca3af); + } + + .gating-stats-card__body { + padding: 1rem; + display: flex; + flex-direction: column; + gap: 1rem; + } + + .gating-stats-card__section { + display: flex; + flex-direction: column; + gap: 0.5rem; + } + .gating-stats-card__section-title { + margin: 0; + font-size: 0.75rem; + font-weight: 500; + color: var(--text-secondary, #6b7280); + text-transform: uppercase; + letter-spacing: 0.05em; + } + + /* Edge metrics */ + .gating-stats-card__metrics { + display: flex; + align-items: center; + gap: 0.5rem; + } + .gating-stats-card__metric { + display: flex; + flex-direction: column; + align-items: center; + } + .gating-stats-card__metric-label { + font-size: 0.6875rem; + color: var(--text-muted, #9ca3af); + } + .gating-stats-card__metric-value { + font-size: 1.125rem; + font-weight: 600; + font-variant-numeric: tabular-nums; + color: var(--text-primary, #111827); + } + .gating-stats-card__metric-arrow { + color: var(--text-muted, #9ca3af); + font-family: monospace; + } + + /* Progress bar */ + .gating-stats-card__progress-container { + display: flex; + align-items: center; + gap: 0.75rem; + } + .gating-stats-card__progress { + flex: 1; + height: 6px; + background: var(--bg-tertiary, #e5e7eb); + border-radius: 3px; + overflow: hidden; + } + .gating-stats-card__progress-bar { + height: 100%; + border-radius: 3px; + transition: width 0.3s ease; + } + .gating-stats-card__progress-bar--reduction { + background: linear-gradient(90deg, #22c55e, #16a34a); + } + .gating-stats-card__progress-label { + font-size: 0.75rem; + font-weight: 500; + color: #15803d; + white-space: nowrap; + } + + /* Verdict stats */ + .gating-stats-card__verdict-stats { + display: flex; + gap: 1rem; + } + .gating-stats-card__verdict-stat { + display: flex; + flex-direction: column; + align-items: center; + padding: 0.5rem 0.75rem; + border-radius: 6px; + min-width: 60px; + } + .gating-stats-card__verdict-stat--surfaced { + background: #dcfce7; + } + .gating-stats-card__verdict-stat--damped { + background: #f3f4f6; + } + .gating-stats-card__verdict-stat--total { + background: #dbeafe; + } + .gating-stats-card__verdict-icon { + font-family: monospace; + font-weight: 700; + font-size: 0.875rem; + } + .gating-stats-card__verdict-stat--surfaced .gating-stats-card__verdict-icon { color: #15803d; } + .gating-stats-card__verdict-stat--damped .gating-stats-card__verdict-icon { color: #6b7280; } + .gating-stats-card__verdict-stat--total .gating-stats-card__verdict-icon { color: #1e40af; } + .gating-stats-card__verdict-value { + font-size: 1rem; + font-weight: 600; + font-variant-numeric: tabular-nums; + } + .gating-stats-card__verdict-label { + font-size: 0.6875rem; + color: var(--text-secondary, #6b7280); + } + + /* Ratio bar */ + .gating-stats-card__ratio-bar { + display: flex; + height: 8px; + border-radius: 4px; + overflow: hidden; + } + .gating-stats-card__ratio-segment { + min-width: 2px; + transition: flex-grow 0.3s ease; + } + .gating-stats-card__ratio-segment--surfaced { + background: #22c55e; + } + .gating-stats-card__ratio-segment--damped { + background: #9ca3af; + } + + /* Duration */ + .gating-stats-card__duration { + display: flex; + justify-content: space-between; + padding-top: 0.5rem; + border-top: 1px solid var(--border-color, #e5e7eb); + font-size: 0.75rem; + } + .gating-stats-card__duration-label { + color: var(--text-muted, #9ca3af); + } + .gating-stats-card__duration-value { + font-family: var(--font-mono, monospace); + color: var(--text-secondary, #6b7280); + } + + /* Aggregated */ + .gating-stats-card__aggregated { + padding-top: 0.75rem; + border-top: 1px solid var(--border-color, #e5e7eb); + } + .gating-stats-card__agg-stats { + display: flex; + gap: 1rem; + margin-top: 0.5rem; + } + .gating-stats-card__agg-stat { + display: flex; + flex-direction: column; + align-items: center; + } + .gating-stats-card__agg-value { + font-size: 0.9375rem; + font-weight: 600; + font-variant-numeric: tabular-nums; + } + .gating-stats-card__agg-label { + font-size: 0.625rem; + color: var(--text-muted, #9ca3af); + } + `], +}) +export class GatingStatisticsCardComponent { + /** Single snapshot statistics */ + readonly statistics = input(null); + + /** Aggregated statistics (for overview) */ + readonly aggregatedStats = input(null); + + /** Card title */ + readonly title = input('Gating Statistics'); + + /** Whether to show aggregated section */ + readonly showAggregated = input(false); + + // Computed values for single snapshot + readonly edgesOriginal = computed(() => this.statistics()?.originalEdgeCount ?? 0); + readonly edgesDeduped = computed(() => this.statistics()?.deduplicatedEdgeCount ?? 0); + readonly edgeReductionPercent = computed(() => this.statistics()?.edgeReductionPercent ?? 0); + readonly verdictsTotal = computed(() => this.statistics()?.totalVerdictCount ?? 0); + readonly verdictsSurfaced = computed(() => this.statistics()?.surfacedVerdictCount ?? 0); + readonly verdictsDamped = computed(() => this.statistics()?.dampedVerdictCount ?? 0); + readonly duration = computed(() => this.statistics()?.duration); + + /** Computed timestamp from aggregated stats */ + readonly computedAt = computed(() => this.aggregatedStats()?.computedAt); + + /** Formatted computed at timestamp */ + readonly formattedComputedAt = computed(() => { + const ts = this.computedAt(); + if (!ts) return ''; + try { + return new Date(ts).toLocaleString(); + } catch { + return ts; + } + }); +} diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/index.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/index.ts new file mode 100644 index 000000000..920930af5 --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/index.ts @@ -0,0 +1,10 @@ +// ----------------------------------------------------------------------------- +// index.ts +// Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui +// Description: Barrel exports for noise-gating components. +// ----------------------------------------------------------------------------- + +export { NoiseGatingSummaryStripComponent } from './noise-gating-summary-strip.component'; +export { DeltaEntryCardComponent } from './delta-entry-card.component'; +export { NoiseGatingDeltaReportComponent } from './noise-gating-delta-report.component'; +export { GatingStatisticsCardComponent } from './gating-statistics-card.component'; diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-delta-report.component.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-delta-report.component.ts new file mode 100644 index 000000000..2f68f348e --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-delta-report.component.ts @@ -0,0 +1,393 @@ +// ----------------------------------------------------------------------------- +// noise-gating-delta-report.component.ts +// Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui +// Task: NG-FE-006 - NoiseGatingDeltaReportComponent +// Description: Container component for displaying noise-gating delta reports. +// Uses tabs for section navigation with summary strip and entry cards. +// ----------------------------------------------------------------------------- + +import { Component, input, output, computed, signal } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { + NoiseGatingDeltaReport, + NoiseGatingDeltaEntry, + NoiseGatingDeltaSection, + getSectionLabel, + groupEntriesBySection, + getSectionPriority, +} from '../../../../core/api/noise-gating.models'; +import { NoiseGatingSummaryStripComponent } from './noise-gating-summary-strip.component'; +import { DeltaEntryCardComponent } from './delta-entry-card.component'; + +/** Tab definition for section navigation */ +interface SectionTab { + section: NoiseGatingDeltaSection; + label: string; + count: number; +} + +/** + * Container component for noise-gating delta reports. + * Provides tabbed navigation through delta sections with entry cards. + */ +@Component({ + selector: 'app-noise-gating-delta-report', + standalone: true, + imports: [ + CommonModule, + NoiseGatingSummaryStripComponent, + DeltaEntryCardComponent, + ], + template: ` +
+ +
+
+

Delta Report

+ @if (report()) { + {{ report()!.reportId }} + } +
+ + @if (report()?.generatedAt) { +
+ Generated: {{ formattedGeneratedAt() }} +
+ } +
+ + + @if (report()) { + + } + + + + + +
+ @if (loading()) { +
+ + Loading delta report... +
+ } @else if (error()) { +
+ ! + {{ error() }} +
+ } @else if (!report()) { +
+ No delta report available + + Select two snapshots to compare + +
+ } @else if (filteredEntries().length === 0) { +
+ No entries in this section +
+ } @else { +
+ @for (entry of filteredEntries(); track entry.vulnerabilityId + entry.productKey) { + + } +
+ } +
+ + + @if (report()) { +
+
+ + From: {{ truncateDigest(report()!.fromSnapshotDigest) }} + + + To: {{ truncateDigest(report()!.toSnapshotDigest) }} + +
+
+ } +
+ `, + styles: [` + .ng-delta-report { + display: flex; + flex-direction: column; + gap: 1rem; + padding: 1rem; + background: var(--bg-primary, #fff); + border: 1px solid var(--border-color, #e5e7eb); + border-radius: 12px; + max-height: 100%; + overflow: hidden; + } + + .ng-delta-report__header { + display: flex; + justify-content: space-between; + align-items: flex-start; + flex-wrap: wrap; + gap: 0.5rem; + } + .ng-delta-report__title { + display: flex; + align-items: baseline; + gap: 0.75rem; + } + .ng-delta-report__title h2 { + margin: 0; + font-size: 1.125rem; + font-weight: 600; + color: var(--text-primary, #111827); + } + .ng-delta-report__id { + font-size: 0.75rem; + color: var(--text-muted, #9ca3af); + font-family: var(--font-mono, monospace); + } + .ng-delta-report__meta { + font-size: 0.75rem; + color: var(--text-muted, #9ca3af); + } + + /* Tabs */ + .ng-delta-report__tabs { + display: flex; + gap: 0.25rem; + border-bottom: 1px solid var(--border-color, #e5e7eb); + padding-bottom: 0.5rem; + overflow-x: auto; + } + .ng-delta-report__tab { + display: flex; + align-items: center; + gap: 0.375rem; + padding: 0.5rem 0.75rem; + background: transparent; + border: none; + border-radius: 6px; + cursor: pointer; + font-size: 0.8125rem; + color: var(--text-secondary, #6b7280); + transition: background-color 0.15s, color 0.15s; + white-space: nowrap; + } + .ng-delta-report__tab:hover:not(.empty) { + background: var(--bg-secondary, #f3f4f6); + color: var(--text-primary, #111827); + } + .ng-delta-report__tab.active { + background: var(--primary-color, #3b82f6); + color: #fff; + } + .ng-delta-report__tab.empty { + opacity: 0.4; + cursor: default; + } + .ng-delta-report__tab-label { + font-weight: 500; + } + .ng-delta-report__tab-count { + padding: 0.125rem 0.375rem; + background: rgba(0, 0, 0, 0.1); + border-radius: 9999px; + font-size: 0.6875rem; + font-weight: 600; + } + .ng-delta-report__tab.active .ng-delta-report__tab-count { + background: rgba(255, 255, 255, 0.25); + } + + /* Content */ + .ng-delta-report__content { + flex: 1; + overflow-y: auto; + min-height: 200px; + } + .ng-delta-report__entries { + display: flex; + flex-direction: column; + gap: 0.75rem; + } + + /* States */ + .ng-delta-report__loading, + .ng-delta-report__error, + .ng-delta-report__empty, + .ng-delta-report__no-entries { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 0.5rem; + padding: 2rem; + text-align: center; + color: var(--text-muted, #9ca3af); + } + .ng-delta-report__spinner { + width: 24px; + height: 24px; + border: 2px solid var(--border-color, #e5e7eb); + border-top-color: var(--primary-color, #3b82f6); + border-radius: 50%; + animation: spin 0.8s linear infinite; + } + @keyframes spin { + to { transform: rotate(360deg); } + } + .ng-delta-report__error { + color: var(--error-color, #dc2626); + } + .ng-delta-report__error-icon { + width: 24px; + height: 24px; + border-radius: 50%; + background: #fee2e2; + color: #dc2626; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + } + .ng-delta-report__empty-hint { + font-size: 0.75rem; + } + + /* Footer */ + .ng-delta-report__footer { + padding-top: 0.75rem; + border-top: 1px solid var(--border-color, #e5e7eb); + } + .ng-delta-report__snapshot-info { + display: flex; + flex-wrap: wrap; + gap: 1rem; + font-size: 0.75rem; + color: var(--text-muted, #9ca3af); + } + .ng-delta-report__snapshot code { + font-family: var(--font-mono, monospace); + background: var(--bg-secondary, #f3f4f6); + padding: 0.125rem 0.375rem; + border-radius: 3px; + } + `], +}) +export class NoiseGatingDeltaReportComponent { + /** The delta report to display */ + readonly report = input(null); + + /** Loading state */ + readonly loading = input(false); + + /** Error message */ + readonly error = input(null); + + /** Whether to show damped section */ + readonly showDamped = input(true); + + /** Whether to show evidence changes section */ + readonly showEvidenceChanges = input(true); + + /** Emits when an entry card is clicked */ + readonly entryClick = output(); + + /** Currently active section */ + readonly activeSection = signal('new'); + + /** Available tabs based on report content */ + readonly tabs = computed(() => { + const r = this.report(); + if (!r) return []; + + const allSections: SectionTab[] = [ + { section: 'new', label: 'New', count: r.summary.newCount }, + { section: 'resolved', label: 'Resolved', count: r.summary.resolvedCount }, + { section: 'confidence_up', label: 'Conf+', count: r.summary.confidenceUpCount }, + { section: 'confidence_down', label: 'Conf-', count: r.summary.confidenceDownCount }, + { section: 'policy_impact', label: 'Policy', count: r.summary.policyImpactCount }, + ]; + + if (this.showDamped()) { + allSections.push({ section: 'damped', label: 'Damped', count: r.summary.dampedCount }); + } + if (this.showEvidenceChanges()) { + allSections.push({ section: 'evidence_changed', label: 'Evidence', count: r.summary.evidenceChangedCount }); + } + + // Sort by priority and filter out empty if configured + return allSections.sort((a, b) => getSectionPriority(a.section) - getSectionPriority(b.section)); + }); + + /** Entries filtered by active section */ + readonly filteredEntries = computed(() => { + const r = this.report(); + if (!r) return []; + + const grouped = groupEntriesBySection(r.entries); + return grouped.get(this.activeSection()) ?? []; + }); + + /** Formatted generated timestamp */ + readonly formattedGeneratedAt = computed(() => { + const ts = this.report()?.generatedAt; + if (!ts) return ''; + try { + return new Date(ts).toLocaleString(); + } catch { + return ts; + } + }); + + /** Handle section selection */ + onSectionSelect(section: NoiseGatingDeltaSection): void { + this.activeSection.set(section); + } + + /** Handle entry card click */ + onEntryClick(entry: NoiseGatingDeltaEntry): void { + this.entryClick.emit(entry); + } + + /** Truncate digest for display */ + truncateDigest(digest: string): string { + if (!digest) return ''; + if (digest.length <= 20) return digest; + // Format: sha256:abc123...def456 + const prefix = digest.slice(0, 12); + const suffix = digest.slice(-6); + return `${prefix}...${suffix}`; + } +} diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-summary-strip.component.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-summary-strip.component.ts new file mode 100644 index 000000000..ae4777f62 --- /dev/null +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/noise-gating/noise-gating-summary-strip.component.ts @@ -0,0 +1,305 @@ +// ----------------------------------------------------------------------------- +// noise-gating-summary-strip.component.ts +// Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui +// Task: NG-FE-004 - NoiseGatingSummaryStripComponent +// Description: Summary strip showing delta section counts for noise-gating reports. +// Follows DeltaSummaryStripComponent pattern with noise-gating-specific sections. +// ----------------------------------------------------------------------------- + +import { Component, input, computed, output } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { + NoiseGatingDeltaSummary, + NoiseGatingDeltaSection, + getSectionLabel, +} from '../../../../core/api/noise-gating.models'; + +/** + * Summary strip component for noise-gating delta reports. + * Displays section counts as interactive badges. + */ +@Component({ + selector: 'app-noise-gating-summary-strip', + standalone: true, + imports: [CommonModule], + template: ` +
+
+ + + + + + + + + + + + + + + + @if (showDamped()) { + + + } + + @if (showEvidenceChanges()) { + + + } +
+ +
+ + Total: + {{ totalCount() }} + + @if (hasActionableChanges()) { + + Action needed + + } +
+
+ `, + styles: [` + .ng-summary-strip { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: var(--bg-secondary, #fff); + border: 1px solid var(--border-color, #e5e7eb); + border-radius: 8px; + flex-wrap: wrap; + gap: 0.75rem; + } + .ng-summary-strip__counts { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + } + .ng-badge { + display: inline-flex; + align-items: center; + gap: 0.25rem; + padding: 0.375rem 0.625rem; + border-radius: 9999px; + font-size: 0.8125rem; + font-weight: 500; + cursor: pointer; + border: 1px solid transparent; + transition: opacity 0.15s, transform 0.15s, box-shadow 0.15s; + } + .ng-badge:hover:not(.empty) { transform: scale(1.03); } + .ng-badge.empty { opacity: 0.4; cursor: default; } + .ng-badge.active { + box-shadow: 0 0 0 2px var(--primary-color, #3b82f6); + transform: scale(1.05); + } + .ng-badge__icon { + font-weight: 700; + font-size: 0.875rem; + font-family: monospace; + } + .ng-badge__count { + font-variant-numeric: tabular-nums; + min-width: 1.25ch; + text-align: center; + } + .ng-badge__label { + font-size: 0.6875rem; + font-weight: 400; + text-transform: lowercase; + } + + /* Section colors */ + .ng-badge--new { + background: #dcfce7; + color: #15803d; + border-color: #86efac; + } + .ng-badge--resolved { + background: #dbeafe; + color: #1e40af; + border-color: #93c5fd; + } + .ng-badge--confidence-up { + background: #ccfbf1; + color: #0d9488; + border-color: #5eead4; + } + .ng-badge--confidence-down { + background: #ffedd5; + color: #c2410c; + border-color: #fdba74; + } + .ng-badge--policy { + background: #fee2e2; + color: #dc2626; + border-color: #fca5a5; + } + .ng-badge--damped { + background: #f3f4f6; + color: #6b7280; + border-color: #d1d5db; + } + .ng-badge--evidence { + background: #f3e8ff; + color: #7c3aed; + border-color: #c4b5fd; + } + + .ng-summary-strip__meta { + display: flex; + align-items: center; + gap: 1rem; + } + .ng-summary-strip__total { + display: flex; + align-items: center; + gap: 0.375rem; + font-size: 0.875rem; + } + .ng-summary-strip__total-label { color: var(--text-muted, #6b7280); } + .ng-summary-strip__total-count { font-weight: 600; } + .ng-summary-strip__actionable { + padding: 0.25rem 0.5rem; + background: #fef3c7; + color: #92400e; + border-radius: 4px; + font-size: 0.75rem; + font-weight: 500; + } + `], +}) +export class NoiseGatingSummaryStripComponent { + /** The delta summary to display */ + readonly summary = input(null); + + /** Whether to show damped section */ + readonly showDamped = input(true); + + /** Whether to show evidence changes section */ + readonly showEvidenceChanges = input(true); + + /** Whether the report has actionable changes */ + readonly hasActionableChanges = input(false); + + /** Currently active/selected section */ + readonly activeSection = input(null); + + /** Emits when a section badge is clicked */ + readonly sectionClick = output(); + + /** Total count of all entries */ + readonly totalCount = computed(() => { + const s = this.summary(); + if (!s) return 0; + return s.totalCount; + }); + + /** Handle section badge click */ + onSectionClick(section: NoiseGatingDeltaSection): void { + const count = this.getSectionCount(section); + if (count > 0) { + this.sectionClick.emit(section); + } + } + + /** Get count for a specific section */ + private getSectionCount(section: NoiseGatingDeltaSection): number { + const s = this.summary(); + if (!s) return 0; + + switch (section) { + case 'new': return s.newCount; + case 'resolved': return s.resolvedCount; + case 'confidence_up': return s.confidenceUpCount; + case 'confidence_down': return s.confidenceDownCount; + case 'policy_impact': return s.policyImpactCount; + case 'damped': return s.dampedCount; + case 'evidence_changed': return s.evidenceChangedCount; + default: return 0; + } + } +} diff --git a/src/Web/StellaOps.Web/src/app/features/triage/components/triage-canvas/triage-canvas.component.ts b/src/Web/StellaOps.Web/src/app/features/triage/components/triage-canvas/triage-canvas.component.ts index 986790bc1..2a0c56ae3 100644 --- a/src/Web/StellaOps.Web/src/app/features/triage/components/triage-canvas/triage-canvas.component.ts +++ b/src/Web/StellaOps.Web/src/app/features/triage/components/triage-canvas/triage-canvas.component.ts @@ -22,8 +22,21 @@ import { VulnerabilityListService, type Vulnerability, type VulnerabilityFilter import { AdvisoryAiService, type AiRecommendation } from '../../services/advisory-ai.service'; import { VexDecisionService, type VexDecision } from '../../services/vex-decision.service'; +// Noise-gating delta report integration (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) +import { NoiseGatingDeltaReportComponent } from '../noise-gating/noise-gating-delta-report.component'; +import { GatingStatisticsCardComponent } from '../noise-gating/gating-statistics-card.component'; +import { + NoiseGatingApiClient, + NOISE_GATING_API_CLIENT, +} from '../../../../core/api/noise-gating.client'; +import { + NoiseGatingDeltaReport, + NoiseGatingDeltaEntry, + GatingStatistics, +} from '../../../../core/api/noise-gating.models'; + export type CanvasPaneMode = 'list' | 'split' | 'detail'; -export type CanvasDetailTab = 'overview' | 'reachability' | 'ai' | 'history' | 'evidence'; +export type CanvasDetailTab = 'overview' | 'reachability' | 'ai' | 'history' | 'evidence' | 'delta'; interface CanvasLayout { leftPaneWidth: number; @@ -34,7 +47,7 @@ interface CanvasLayout { @Component({ selector: 'app-triage-canvas', standalone: true, - imports: [CommonModule, RouterLink], + imports: [CommonModule, RouterLink, NoiseGatingDeltaReportComponent, GatingStatisticsCardComponent], template: `
@@ -413,6 +426,26 @@ interface CanvasLayout {
} + @case ('delta') { +
+
+
+ +
+ +
+
+ } } @@ -1137,6 +1170,41 @@ interface CanvasLayout { margin-top: 0.5rem; } + /* Delta Panel Layout (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) */ + .detail-panel--delta { + padding: 0; + } + + .delta-panel-layout { + display: grid; + grid-template-columns: 1fr 280px; + gap: 1rem; + height: 100%; + padding: 1rem; + } + + .delta-panel-layout__main { + overflow-y: auto; + min-height: 0; + } + + .delta-panel-layout__sidebar { + display: flex; + flex-direction: column; + gap: 1rem; + } + + @media (max-width: 1024px) { + .delta-panel-layout { + grid-template-columns: 1fr; + grid-template-rows: auto 1fr; + } + + .delta-panel-layout__sidebar { + order: -1; + } + } + /* Responsive */ @media (max-width: 768px) { .triage-canvas--split .triage-canvas__list-pane { @@ -1162,6 +1230,9 @@ export class TriageCanvasComponent implements OnInit, OnDestroy { private readonly route = inject(ActivatedRoute); private readonly router = inject(Router); + // Noise-gating delta integration (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) + private readonly noiseGatingClient = inject(NOISE_GATING_API_CLIENT, { optional: true }); + private subscriptions: Subscription[] = []; private resizing = false; @@ -1178,6 +1249,7 @@ export class TriageCanvasComponent implements OnInit, OnDestroy { { id: 'ai', label: 'AI Analysis' }, { id: 'history', label: 'VEX History' }, { id: 'evidence', label: 'Evidence' }, + { id: 'delta', label: 'Delta' }, ]; readonly layout = signal({ @@ -1198,6 +1270,12 @@ export class TriageCanvasComponent implements OnInit, OnDestroy { return this.aiService.getCachedRecommendations(selected.id) ?? []; }); + // Delta report signals (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) + readonly deltaReport = signal(null); + readonly deltaLoading = signal(false); + readonly deltaError = signal(null); + readonly gatingStatistics = signal(null); + private keyboardStatusTimeout: ReturnType | null = null; ngOnInit(): void { @@ -1265,8 +1343,13 @@ export class TriageCanvasComponent implements OnInit, OnDestroy { case 'Escape': this.clearBulkSelection(); break; + case 'd': + event.preventDefault(); + this.setDetailTab('delta'); + this.announceStatus('Delta Report'); + break; case '?': - this.announceStatus('N: Next, P: Prev, M: Mark Not Affected, A: Analyze, V: VEX'); + this.announceStatus('N: Next, P: Prev, M: Mark Not Affected, A: Analyze, V: VEX, D: Delta'); break; } } @@ -1291,6 +1374,11 @@ export class TriageCanvasComponent implements OnInit, OnDestroy { setDetailTab(tab: CanvasDetailTab): void { this.activeDetailTab.set(tab); + + // Load delta report when switching to delta tab (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) + if (tab === 'delta' && !this.deltaReport() && !this.deltaLoading()) { + this.loadDeltaReport(); + } } isSeverityActive(severity: string): boolean { @@ -1464,6 +1552,65 @@ export class TriageCanvasComponent implements OnInit, OnDestroy { ); } + // Delta report methods (Sprint: SPRINT_20260104_002_FE_noise_gating_delta_ui) + onDeltaEntryClick(entry: NoiseGatingDeltaEntry): void { + // Navigate to the specific vulnerability if it's in our list + const items = this.vulnService.items(); + const matchingVuln = items.find(v => v.cveId === entry.vulnerabilityId); + if (matchingVuln) { + this.selectVulnerability(matchingVuln); + this.activeDetailTab.set('overview'); + } else { + // Show the entry details in a status message + this.announceStatus(`${entry.vulnerabilityId}: ${entry.section}`); + } + } + + loadDeltaReport(): void { + if (!this.noiseGatingClient) { + this.deltaError.set('Delta report API not available'); + return; + } + + this.deltaLoading.set(true); + this.deltaError.set(null); + + // For demo: use mock snapshot IDs - in real usage these would come from scan context + const fromSnapshotId = 'snapshot-previous'; + const toSnapshotId = 'snapshot-current'; + + this.subscriptions.push( + this.noiseGatingClient.computeDelta(fromSnapshotId, toSnapshotId).subscribe({ + next: (report) => { + this.deltaReport.set(report); + this.deltaLoading.set(false); + }, + error: (err) => { + this.deltaError.set(err?.message ?? 'Failed to load delta report'); + this.deltaLoading.set(false); + }, + }) + ); + + // Also load gating statistics + this.loadGatingStatistics(); + } + + private loadGatingStatistics(): void { + if (!this.noiseGatingClient) return; + + this.subscriptions.push( + this.noiseGatingClient.getGatingStatistics().subscribe({ + next: (stats) => { + this.gatingStatistics.set(stats); + }, + error: (err) => { + console.error('Failed to load gating statistics:', err); + }, + }) + ); + } + private selectNextVuln(): void { const items = this.vulnService.items(); const current = this.vulnService.selectedItem(); diff --git a/src/__Libraries/StellaOps.ReachGraph/Deduplication/DeduplicatedEdge.cs b/src/__Libraries/StellaOps.ReachGraph/Deduplication/DeduplicatedEdge.cs new file mode 100644 index 000000000..b67d3dd5a --- /dev/null +++ b/src/__Libraries/StellaOps.ReachGraph/Deduplication/DeduplicatedEdge.cs @@ -0,0 +1,138 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using StellaOps.ReachGraph.Schema; + +namespace StellaOps.ReachGraph.Deduplication; + +/// +/// An edge that has been deduplicated from multiple source edges. +/// Preserves provenance by tracking all contributing sources. +/// +public sealed record DeduplicatedEdge +{ + /// + /// Gets the semantic key for this edge. + /// + public required EdgeSemanticKey Key { get; init; } + + /// + /// Gets the source node ID (from entry point). + /// + public required string From { get; init; } + + /// + /// Gets the target node ID (to sink). + /// + public required string To { get; init; } + + /// + /// Gets the aggregated explanation for this edge. + /// + public required EdgeExplanation Why { get; init; } + + /// + /// Gets the set of source identifiers that contributed this edge. + /// + public required ImmutableHashSet Sources { get; init; } + + /// + /// Gets the maximum strength (weight) among all contributing sources. + /// + public required double Strength { get; init; } + + /// + /// Gets the timestamp of the most recent observation of this edge. + /// + public required DateTimeOffset LastSeen { get; init; } + + /// + /// Gets the number of contributing sources. + /// + public int SourceCount => Sources.Count; + + /// + /// Gets whether this edge has multiple confirming sources. + /// + public bool IsCorroborated => Sources.Count > 1; +} + +/// +/// Builder for creating instances by merging multiple source edges. +/// +public sealed class DeduplicatedEdgeBuilder +{ + private readonly EdgeSemanticKey _key; + private readonly string _from; + private readonly string _to; + private readonly HashSet _sources = new(StringComparer.Ordinal); + private EdgeExplanation? _explanation; + private double _maxStrength; + private DateTimeOffset _lastSeen = DateTimeOffset.MinValue; + + /// + /// Initializes a new builder for the given semantic key. + /// + public DeduplicatedEdgeBuilder(EdgeSemanticKey key, string from, string to) + { + _key = key; + _from = from; + _to = to; + } + + /// + /// Adds a source edge to this builder. + /// + /// The source identifier (e.g., feed name, analyzer ID). + /// The edge explanation from this source. + /// The strength/weight from this source. + /// When this source observed the edge. + /// This builder for chaining. + public DeduplicatedEdgeBuilder AddSource( + string sourceId, + EdgeExplanation explanation, + double strength, + DateTimeOffset observedAt) + { + _sources.Add(sourceId); + + // Keep the strongest explanation + if (strength > _maxStrength || _explanation is null) + { + _maxStrength = strength; + _explanation = explanation; + } + + // Track most recent observation + if (observedAt > _lastSeen) + { + _lastSeen = observedAt; + } + + return this; + } + + /// + /// Builds the deduplicated edge. + /// + /// The deduplicated edge with merged provenance. + /// If no sources were added. + public DeduplicatedEdge Build() + { + if (_sources.Count == 0 || _explanation is null) + { + throw new InvalidOperationException("At least one source must be added before building."); + } + + return new DeduplicatedEdge + { + Key = _key, + From = _from, + To = _to, + Why = _explanation, + Sources = _sources.ToImmutableHashSet(StringComparer.Ordinal), + Strength = _maxStrength, + LastSeen = _lastSeen + }; + } +} diff --git a/src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeDeduplicator.cs b/src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeDeduplicator.cs new file mode 100644 index 000000000..cdc40aa91 --- /dev/null +++ b/src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeDeduplicator.cs @@ -0,0 +1,137 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Collections.Immutable; +using StellaOps.ReachGraph.Schema; + +namespace StellaOps.ReachGraph.Deduplication; + +/// +/// Service for deduplicating edges from multiple sources into semantically unique edges. +/// +public interface IEdgeDeduplicator +{ + /// + /// Deduplicates a collection of edges by their semantic keys. + /// + /// The edges to deduplicate. + /// Function to extract semantic key from an edge. + /// Function to extract source ID from an edge. + /// Function to extract strength/weight from an edge. + /// Function to extract observation timestamp. + /// Deduplicated edges with merged provenance. + IReadOnlyList Deduplicate( + IEnumerable edges, + Func keyExtractor, + Func sourceExtractor, + Func strengthExtractor, + Func timestampExtractor); +} + +/// +/// Default implementation of . +/// +public sealed class EdgeDeduplicator : IEdgeDeduplicator +{ + /// + /// Gets the singleton instance. + /// + public static IEdgeDeduplicator Instance { get; } = new EdgeDeduplicator(); + + /// + public IReadOnlyList Deduplicate( + IEnumerable edges, + Func keyExtractor, + Func sourceExtractor, + Func strengthExtractor, + Func timestampExtractor) + { + ArgumentNullException.ThrowIfNull(edges); + ArgumentNullException.ThrowIfNull(keyExtractor); + ArgumentNullException.ThrowIfNull(sourceExtractor); + ArgumentNullException.ThrowIfNull(strengthExtractor); + ArgumentNullException.ThrowIfNull(timestampExtractor); + + // Group edges by semantic key + var builders = new Dictionary(); + + foreach (var edge in edges) + { + var key = keyExtractor(edge); + + if (!builders.TryGetValue(key, out var builder)) + { + builder = new DeduplicatedEdgeBuilder(key, edge.From, edge.To); + builders[key] = builder; + } + + builder.AddSource( + sourceExtractor(edge), + edge.Why, + strengthExtractor(edge), + timestampExtractor(edge)); + } + + // Build deduplicated edges, sorted by strength descending for stability + return builders.Values + .Select(b => b.Build()) + .OrderByDescending(e => e.Strength) + .ThenBy(e => e.Key.ComputeKey(), StringComparer.Ordinal) + .ToList(); + } +} + +/// +/// Extensions for edge deduplication. +/// +public static class EdgeDeduplicatorExtensions +{ + /// + /// Deduplicates edges using default extractors based on edge properties. + /// + /// The deduplicator instance. + /// The edges to deduplicate. + /// The vulnerability ID to associate with edges. + /// Default source ID if not specified. + /// Time provider for timestamps. + /// Deduplicated edges. + public static IReadOnlyList DeduplicateWithDefaults( + this IEdgeDeduplicator deduplicator, + IEnumerable edges, + string vulnerabilityId, + string defaultSource = "unknown", + TimeProvider? timeProvider = null) + { + var time = timeProvider ?? TimeProvider.System; + var now = time.GetUtcNow(); + + return deduplicator.Deduplicate( + edges, + keyExtractor: e => new EdgeSemanticKey(e.From, e.To, vulnerabilityId), + sourceExtractor: _ => defaultSource, + strengthExtractor: e => GetEdgeStrength(e.Why), + timestampExtractor: _ => now); + } + + private static double GetEdgeStrength(EdgeExplanation explanation) + { + // Use the explanation's confidence as the base strength + // Map edge explanation type to a multiplier + var typeMultiplier = explanation.Type switch + { + EdgeExplanationType.DirectCall => 1.0, + EdgeExplanationType.Import => 0.95, + EdgeExplanationType.DynamicLoad => 0.9, + EdgeExplanationType.Ffi => 0.85, + EdgeExplanationType.Reflection => 0.8, + EdgeExplanationType.LoaderRule => 0.75, + EdgeExplanationType.TaintGate => 0.7, + EdgeExplanationType.EnvGuard => 0.65, + EdgeExplanationType.FeatureFlag => 0.6, + EdgeExplanationType.PlatformArch => 0.6, + EdgeExplanationType.Unknown => 0.5, + _ => 0.5 + }; + + return explanation.Confidence * typeMultiplier; + } +} diff --git a/src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeSemanticKey.cs b/src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeSemanticKey.cs new file mode 100644 index 000000000..b37f40dcf --- /dev/null +++ b/src/__Libraries/StellaOps.ReachGraph/Deduplication/EdgeSemanticKey.cs @@ -0,0 +1,134 @@ +// Licensed to StellaOps under the AGPL-3.0-or-later license. + +using System.Globalization; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.ReachGraph.Deduplication; + +/// +/// A semantic key for edge deduplication that identifies edges with equivalent meaning +/// regardless of their source or representation. +/// +/// +/// Edges from different sources (reachability analysis, call graph, binary analysis) +/// may represent the same semantic relationship. This key normalizes them for deduplication. +/// +/// Two edges are semantically equivalent if they have the same: +/// - Entry point node ID +/// - Sink node ID +/// - Vulnerability ID (if applicable) +/// - Applied gate (if any) +/// +public readonly record struct EdgeSemanticKey : IEquatable +{ + /// + /// Gets the entry point node identifier. + /// + public string EntryPointId { get; } + + /// + /// Gets the sink (vulnerable) node identifier. + /// + public string SinkId { get; } + + /// + /// Gets the vulnerability identifier, if this edge is associated with one. + /// + public string? VulnerabilityId { get; } + + /// + /// Gets the applied gate identifier, if any gate was applied to this edge. + /// + public string? GateApplied { get; } + + /// + /// Initializes a new instance of the struct. + /// + /// The entry point node ID. + /// The sink node ID. + /// Optional vulnerability ID. + /// Optional gate identifier. + public EdgeSemanticKey( + string entryPointId, + string sinkId, + string? vulnerabilityId = null, + string? gateApplied = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(entryPointId); + ArgumentException.ThrowIfNullOrWhiteSpace(sinkId); + + EntryPointId = entryPointId; + SinkId = sinkId; + VulnerabilityId = NormalizeId(vulnerabilityId); + GateApplied = gateApplied; + } + + /// + /// Computes a canonical string key for this semantic key. + /// + /// A canonical string representation suitable for dictionary keys. + public string ComputeKey() + { + var builder = new StringBuilder(256); + builder.Append(EntryPointId); + builder.Append('|'); + builder.Append(SinkId); + builder.Append('|'); + builder.Append(VulnerabilityId ?? string.Empty); + builder.Append('|'); + builder.Append(GateApplied ?? string.Empty); + return builder.ToString(); + } + + /// + /// Computes a SHA-256 hash of the canonical key for compact storage. + /// + /// A lowercase hex-encoded SHA-256 hash. + public string ComputeHash() + { + var key = ComputeKey(); + var bytes = Encoding.UTF8.GetBytes(key); + var hash = SHA256.HashData(bytes); + return Convert.ToHexStringLower(hash); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine( + EntryPointId, + SinkId, + VulnerabilityId ?? string.Empty, + GateApplied ?? string.Empty); + } + + /// + public bool Equals(EdgeSemanticKey other) + { + return string.Equals(EntryPointId, other.EntryPointId, StringComparison.Ordinal) && + string.Equals(SinkId, other.SinkId, StringComparison.Ordinal) && + string.Equals(VulnerabilityId, other.VulnerabilityId, StringComparison.OrdinalIgnoreCase) && + string.Equals(GateApplied, other.GateApplied, StringComparison.Ordinal); + } + + /// + public override string ToString() => ComputeKey(); + + private static string? NormalizeId(string? id) + { + if (string.IsNullOrWhiteSpace(id)) + { + return null; + } + + // Normalize CVE IDs to uppercase for consistent comparison + if (id.StartsWith("cve-", StringComparison.OrdinalIgnoreCase) || + id.StartsWith("CVE-", StringComparison.Ordinal)) + { + return id.ToUpperInvariant(); + } + + return id; + } +}