From 6e45066e371667ef96fbd4cc99a05223f151f12c Mon Sep 17 00:00:00 2001 From: StellaOps Bot Date: Sat, 13 Dec 2025 09:37:15 +0200 Subject: [PATCH] up --- = | 0 Directory.Build.props | 1 - bench/README.md | 128 +- docs/09_API_CLI_REFERENCE.md | 89 +- docs/api/policy.md | 74 +- ...1_0001_0001_reachability_evidence_chain.md | 50 +- ...3_0001_0001_scanner_java_detection_gaps.md | 45 +- ...0001_0001_scanner_python_detection_gaps.md | 32 +- ...6_0001_0001_scanner_node_detection_gaps.md | 38 +- ...07_0001_0001_scanner_bun_detection_gaps.md | 35 +- ...scanner_language_detection_gaps_program.md | 21 +- ...1_scanner_non_language_scanners_quality.md | 5 +- ...rypoint_detection_reengineering_program.md | 159 +++ ...11_0001_0001_semantic_entrypoint_engine.md | 163 +++ .../SPRINT_0215_0001_0001_vuln_triage_ux.md | 5 +- docs/modules/scanner/README.md | 3 +- docs/modules/scanner/analyzers-bun.md | 81 ++ docs/modules/scanner/analyzers-java.md | 65 + docs/modules/scanner/analyzers-node.md | 79 ++ docs/modules/scanner/analyzers-python.md | 69 ++ docs/modules/scanner/architecture.md | 15 +- .../scanner/language-analyzers-contract.md | 110 ++ docs/policy/dsl.md | 263 +++-- .../binary-reachability-schema.md | 461 ++++++++ docs/reachability/corpus-plan.md | 92 +- .../edge-explainability-schema.md | 416 +++++++ docs/reachability/explainability-schema.md | 454 +++++++ docs/reachability/function-level-evidence.md | 636 +++++++--- docs/reachability/graph-revision-schema.md | 377 ++++++ docs/reachability/hybrid-attestation.md | 94 +- docs/replay/replay-manifest-v2-acceptance.md | 311 +++++ docs/uncertainty/README.md | 100 +- .../SimCryptoService.csproj | 1 - .../sim-crypto-smoke/SimCryptoSmoke.csproj | 1 - .../indices/reachability_store_indices.js | 67 ++ samples/reachability/README.md | 38 + .../reachability/openvex-affected-sample.json | 86 ++ .../openvex-not-affected-sample.json | 68 ++ .../replay-manifest-v2-sample.json | 62 + samples/reachability/richgraph-v1-sample.json | 117 ++ .../reachability/runtime-facts-sample.ndjson | 3 + .../runtime/java-fat-archive/libs/app-fat.jar | Bin 0 -> 1092 bytes .../runtime/node-detection-gaps/package.json | 11 + .../packages/app/package.json | 7 + .../packages/app/src/index.ts | 5 + .../packages/app/src/util.ts | 7 + .../packages/lib/package.json | 4 + .../packages/lib/src/index.ts | 1 + samples/runtime/node-detection-gaps/yarn.lock | 13 + .../ServiceCollectionExtensions.cs | 1 + .../StellaOps.Attestor.Infrastructure.csproj | 1 + .../MessagingAttestorVerificationCache.cs | 107 ++ .../MessagingTokenCache.cs | 83 ++ .../StellaOps.Auth.Client.csproj | 1 + .../Claims/MessagingLdapClaimsCache.cs | 57 + .../StellaOps.Authority.Plugin.Ldap.csproj | 3 +- ...Ops.Authority.Plugin.Standard.Tests.csproj | 4 +- .../StellaOps.Authority.Tests.csproj | 4 +- .../Scanner.Analyzers/README.md | 9 +- .../NodeBenchMetrics.cs | 268 +++++ .../Program.cs | 7 +- .../Reporting/BenchmarkJsonWriter.cs | 2 + .../Reporting/BenchmarkScenarioReport.cs | 9 +- .../Reporting/PrometheusWriter.cs | 19 + .../ScenarioResult.cs | 2 + .../ScenarioRunners.cs | 29 +- .../StellaOps.Bench.ScannerAnalyzers.csproj | 3 +- .../Scanner.Analyzers/baseline.csv | 16 +- .../Scanner.Analyzers/config.json | 36 + .../Scanner.Analyzers/lang/README.md | 2 + src/Bench/StellaOps.Bench/TASKS.md | 2 + .../Services/MessagingAdvisoryChunkCache.cs | 52 + .../StellaOps.Concelier.WebService.csproj | 1 + .../Services/MessagingGraphOverlayCache.cs | 76 ++ .../StellaOps.Excititor.WebService.csproj | 1 + .../StellaOps.Gateway.WebService.Tests.csproj | 2 +- .../Caching/MessagingPolicyEvaluationCache.cs | 202 ++++ ...PolicyEngineServiceCollectionExtensions.cs | 4 +- .../Gates/PolicyGateEvaluator.cs | 2 +- .../MessagingReachabilityFactsOverlayCache.cs | 180 +++ src/Policy/StellaOps.Policy.Engine/TASKS.md | 2 +- .../Tenancy/TenantContextMiddleware.cs | 17 +- .../Migration/PolicyMigrator.cs | 10 +- .../Tenancy/TenantContextTests.cs | 113 +- .../AGENTS.md | 45 + .../BunLanguageAnalyzer.cs | 294 ++++- .../BunDeclaredDependencyCollector.cs | 80 ++ .../Internal/BunEvidenceHasher.cs | 45 + .../Internal/BunInstalledCollector.cs | 36 +- .../Internal/BunLockEntry.cs | 5 +- .../Internal/BunLockParser.cs | 17 +- .../Internal/BunLockScopeClassifier.cs | 203 ++++ .../Internal/BunPackage.cs | 58 +- .../Internal/BunProjectDiscoverer.cs | 135 ++- .../Internal/BunVersionSpec.cs | 144 +++ .../Internal/BunWorkspaceHelper.cs | 80 +- .../TASKS.md | 13 + .../Internal/DenoBundleInspector.cs | 1 + .../Internal/DenoContainerAdapter.cs | 16 +- .../Internal/DenoWorkspaceNormalizer.cs | 4 +- .../Runtime/DenoRuntimeTraceSerializer.cs | 28 +- .../Internal/DotNetEntrypointResolver.cs | 101 +- .../GoLanguageAnalyzer.cs | 2 +- .../Internal/GoBinaryScanner.cs | 39 + .../Internal/GoCapabilityScanner.cs | 4 +- .../Internal/GoCgoDetector.cs | 2 +- .../Internal/GoLicenseDetector.cs | 4 +- .../Internal/GoVersionConflictDetector.cs | 2 +- .../Internal/Jni/JavaJniAnalyzer.cs | 81 +- .../JavaLanguageAnalyzer.cs | 1043 +++++++++++++++-- .../Internal/NodeDependencyIndex.cs | 124 +- .../Internal/NodeImportWalker.cs | 80 +- .../Internal/NodeLockData.cs | 278 ++++- .../Internal/NodeLockEntry.cs | 4 +- .../Internal/NodePackage.cs | 17 + .../Internal/NodePackageCollector.cs | 439 ++++++- .../Internal/NodeWorkspaceIndex.cs | 214 +++- .../NodeLanguageAnalyzer.cs | 272 +++++ .../TASKS.md | 16 + .../Entrypoints/PythonEntrypointDiscovery.cs | 21 +- .../Packaging/Adapters/PipEditableAdapter.cs | 21 +- .../Packaging/PythonPackageDiscovery.cs | 14 +- .../Internal/PythonDistributionVfsLoader.cs | 937 +++++++++++++++ .../PythonInputNormalizer.cs | 152 ++- .../PythonVirtualFileSystem.cs | 82 +- .../PythonLanguageAnalyzer.cs | 463 ++++++-- .../TASKS.md | 14 + .../Observations/RubyObservationBuilder.cs | 4 +- .../Internal/RubyBundlerConfig.cs | 71 +- .../Internal/RubyCapabilityDetector.cs | 3 +- .../Internal/RubyContainerScanner.cs | 43 +- .../Internal/RubyRuntimeGraphBuilder.cs | 5 - .../RubyLanguageAnalyzer.cs | 35 +- .../TASKS.md | 1 + .../Core/LanguageExplicitKey.cs | 26 + .../ReachabilityRichGraphPublisher.cs | 100 +- .../Bun/BunLanguageAnalyzerTests.cs | 95 ++ .../Fixtures/lang/bun/bunfig-only/bunfig.toml | 2 + .../lang/bun/bunfig-only/expected.json | 74 ++ .../lang/bun/bunfig-only/package.json | 11 + .../.layers/layer0/app/bun.lock | 6 + .../.layers/layer0/app/package.json | 7 + .../lang/bun/container-layers/expected.json | 34 + .../lang/bun/custom-registry/expected.json | 15 +- .../Fixtures/lang/bun/deep-tree/expected.json | 28 +- .../lang/bun/git-dependencies/expected.json | 10 +- .../Fixtures/lang/bun/isolated/expected.json | 28 +- .../lang/bun/jsonc-lockfile/expected.json | 15 +- .../bun/lockfile-dev-classification/bun.lock | 10 + .../lockfile-dev-classification/expected.json | 98 ++ .../lockfile-dev-classification/package.json | 11 + .../lang/bun/lockfile-only/expected.json | 12 +- .../lang/bun/multi-workspace/expected.json | 28 +- .../lang/bun/multi-workspace/package.json | 11 + .../lang/bun/non-concrete-versions/bun.lock | 21 + .../bun/non-concrete-versions/expected.json | 80 ++ .../bun/non-concrete-versions/package.json | 10 + .../lang/bun/patched-multi-version/bun.lock | 8 + .../bun/patched-multi-version/expected.json | 86 ++ .../bun/patched-multi-version/package.json | 12 + .../patches/lodash@4.17.20.patch | 8 + .../patches/lodash@4.17.21.patch | 8 + .../lang/bun/patched-packages/expected.json | 15 +- .../lang/bun/scoped-packages/expected.json | 28 +- .../Fixtures/lang/bun/standard/expected.json | 15 +- .../Fixtures/lang/bun/symlinks/expected.json | 15 +- .../lang/bun/workspaces/expected.json | 15 +- .../Parsers/BunLockParserTests.cs | 26 +- .../Parsers/BunLockScopeClassifierTests.cs | 30 + .../Parsers/BunPackageTests.cs | 4 +- .../Parsers/BunWorkspaceHelperTests.cs | 13 +- ...ps.Scanner.Analyzers.Lang.Bun.Tests.csproj | 2 - .../Deno/DenoRuntimePathHasherTests.cs | 4 +- .../Deno/DenoRuntimeTraceProbeTests.cs | 2 +- .../Deno/DenoRuntimeTraceRecorderTests.cs | 4 +- .../Deno/DenoRuntimeTraceSerializerTests.cs | 2 +- .../Golden/DenoAnalyzerGoldenTests.cs | 17 +- ...s.Scanner.Analyzers.Lang.Deno.Tests.csproj | 2 - ...Scanner.Analyzers.Lang.DotNet.Tests.csproj | 3 +- .../Fixtures/lang/go/stripped/expected.json | 34 +- ...Ops.Scanner.Analyzers.Lang.Go.Tests.csproj | 2 - .../java/pomxml-only-jar/expected.json | 35 + .../expected.json | 65 + .../java/war-embedded-maven/expected.json | 65 + .../Java/JavaLanguageAnalyzerTests.cs | 282 ++++- ...s.Scanner.Analyzers.Lang.Java.Tests.csproj | 2 - .../Fixtures/expected.json | 80 ++ .../lang/node/container-env/expected.json | 5 +- .../lang/node/container-layers/expected.json | 77 +- .../layers/layer1/app/package.json | 7 + .../declared-only-package-json/expected.json | 200 ++++ .../declared-only-package-json/package.json | 20 + .../lang/node/entrypoints/expected.json | 3 +- .../lang/node/imports-dynamic/expected.json | 52 +- .../node/lock-only-package-lock/expected.json | 98 ++ .../lock-only-package-lock/package-lock.json | 18 + .../lang/node/lock-only-pnpm/expected.json | 75 ++ .../lang/node/lock-only-pnpm/pnpm-lock.yaml | 10 + .../node/lock-only-yarn-berry/expected.json | 98 ++ .../lang/node/lock-only-yarn-berry/yarn.lock | 23 + .../lang/node/pnpm-store/expected.json | 6 +- .../lang/node/runtime-evidence/expected.json | 3 +- .../Fixtures/lang/node/shebang/expected.json | 3 +- .../lang/node/version-targets/expected.json | 5 +- .../lang/node/workspaces/expected.json | 89 +- .../lang/node/workspaces/package.json | 15 +- .../packages/nested/tool/package.json | 7 + .../Fixtures/lang/node/yarn-pnp/expected.json | 48 +- .../Node/NodeDeterminismTests.cs | 101 ++ .../Node/NodeImportWalkerTests.cs | 39 + .../Node/NodeLanguageAnalyzerTests.cs | 64 + .../Node/NodeLockDataTests.cs | 169 ++- .../NodePackageCollectorTraversalTests.cs | 171 +++ .../Node/NodeWorkspaceIndexTests.cs | 102 ++ ...s.Scanner.Analyzers.Lang.Node.Tests.csproj | 2 - ...ps.Scanner.Analyzers.Lang.Php.Tests.csproj | 2 - .../PythonEntrypointDiscoveryTests.cs | 35 + .../hiddenpkg-0.1.0.dist-info/METADATA | 4 + .../site-packages/hiddenpkg/__init__.py | 1 + .../python/layered-editable/expected.json | 122 +- .../layerspkg-0.2.0.dist-info/METADATA | 4 + .../site-packages/layerspkg/__init__.py | 1 + .../lang/python/pip-cache/expected.json | 5 +- .../lang/python/simple-venv/expected.json | 5 +- .../Python/PythonLanguageAnalyzerTests.cs | 132 +++ ...Scanner.Analyzers.Lang.Python.Tests.csproj | 2 - .../PythonVirtualFileSystemTests.cs | 16 +- .../Fixtures/lang/ruby/cli-app/expected.json | 1 + .../lang/ruby/complex-app/expected.json | 13 +- .../lang/ruby/container-app/expected.json | 6 +- .../lang/ruby/legacy-app/expected.json | 29 +- .../lang/ruby/rails-app/expected.json | 1 + .../lang/ruby/simple-app/expected.json | 1 + .../lang/ruby/sinatra-app/expected.json | 1 + ...s.Scanner.Analyzers.Lang.Ruby.Tests.csproj | 2 - .../Fixtures/lang/deno/full/expected.json | 12 +- .../Fixtures/lang/ruby/basic/expected.json | 8 +- .../lang/ruby/git-sources/expected.json | 7 +- .../lang/ruby/workspace/expected.json | 7 +- .../Fixtures/lang/rust/fallback/expected.json | 8 +- .../lang/rust/heuristics/expected.json | 12 +- .../Rust/RustFixtureBinaries.cs | 61 + .../RustHeuristicCoverageComparisonTests.cs | 1 + .../Rust/RustLanguageAnalyzerTests.cs | 2 + ...llaOps.Scanner.Analyzers.Lang.Tests.csproj | 2 - .../TestUtilities/JavaFixtureBuilder.cs | 127 ++ .../ReachabilityUnionPublisherTests.cs | 13 + .../ReachabilityUnionWriterTests.cs | 8 +- .../Replay/RecordModeAssemblerTests.cs | 9 +- .../StellaOps.Scanner.Core.Tests.csproj | 3 + .../BinaryReachabilityLifterTests.cs | 6 +- .../FakeFileContentAddressableStore.cs | 3 + .../RichGraphPublisherTests.cs | 42 + .../Fixtures/descriptor.baseline.json | 8 +- .../ReachabilityStore/CallEdgeDocument.cs | 30 + .../ReachabilityStore/CveFuncHitDocument.cs | 30 + .../ReachabilityStore/FuncNodeDocument.cs | 40 + .../IReachabilityStoreRepository.cs | 28 + .../InMemoryReachabilityStoreRepository.cs | 250 ++++ src/Signals/StellaOps.Signals/Program.cs | 1 + .../Services/CallgraphIngestionService.cs | 9 + .../Services/ReachabilityScoringService.cs | 42 +- src/Signals/StellaOps.Signals/TASKS.md | 4 +- .../CallgraphIngestionServiceTests.cs | 11 + .../ReachabilityScoringServiceTests.cs | 73 ++ .../StellaOps.Telemetry.Core.Tests.csproj | 1 - .../graph/graph-explorer.component.html | 17 +- .../tests/e2e/a11y-smoke.spec.ts | 2 +- .../Dpop/MessagingDpopReplayCache.cs | 43 + .../ReplayManifestTests.cs | 42 +- .../ReplayManifestV2Tests.cs | 483 ++++++++ .../StellaOps.Replay.Core/CasValidator.cs | 117 ++ .../ReachabilityReplayWriter.cs | 53 +- .../StellaOps.Replay.Core/ReplayManifest.cs | 50 +- .../ReplayManifestValidator.cs | 397 +++++++ .../StellaOps.Microservice.Tests.csproj | 2 +- .../StellaOps.Router.Common.Tests.csproj | 2 +- .../StellaOps.Router.Config.Tests.csproj | 2 +- ...Ops.Router.Transport.InMemory.Tests.csproj | 2 +- ...Ops.Router.Transport.RabbitMq.Tests.csproj | 2 +- .../CorpusFixtureTests.cs | 41 +- .../FixtureCoverageTests.cs | 56 + .../ReachabilityReplayWriterTests.cs | 8 +- .../ReplayMongoModelsTests.cs | 57 - .../ScannerToSignalsReachabilityTests.cs | 70 +- .../ReachabilityScoringTests.cs | 66 +- .../RuntimeFactsIngestionServiceTests.cs | 92 +- tests/reachability/corpus/README.md | 3 +- .../expect.yaml | 11 - .../ground-truth.json | 16 + .../expect.yaml | 11 - .../ground-truth.json | 16 + tests/reachability/corpus/manifest.json | 26 +- .../expect.yaml | 11 - .../ground-truth.json | 16 + .../rust-axum-header-parsing-TBD/expect.yaml | 11 - .../ground-truth.json | 11 + .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../images/reachable/manifest.json | 16 +- .../images/unreachable/manifest.json | 16 +- .../harness/update_variant_manifests.py | 82 ++ tests/reachability/runners/run_all.ps1 | 8 + tests/reachability/runners/run_all.sh | 9 + .../scripts/update_corpus_manifest.py | 4 +- tests/shared/OpenSslAutoInit.cs | 2 +- tests/shared/OpenSslLegacyShim.cs | 2 +- 349 files changed, 17160 insertions(+), 1867 deletions(-) create mode 100644 = create mode 100644 docs/implplan/SPRINT_0410_0001_0001_entrypoint_detection_reengineering_program.md create mode 100644 docs/implplan/SPRINT_0411_0001_0001_semantic_entrypoint_engine.md rename docs/implplan/{ => archived}/SPRINT_0215_0001_0001_vuln_triage_ux.md (97%) create mode 100644 docs/modules/scanner/analyzers-bun.md create mode 100644 docs/modules/scanner/analyzers-java.md create mode 100644 docs/modules/scanner/analyzers-node.md create mode 100644 docs/modules/scanner/analyzers-python.md create mode 100644 docs/modules/scanner/language-analyzers-contract.md create mode 100644 docs/reachability/binary-reachability-schema.md create mode 100644 docs/reachability/edge-explainability-schema.md create mode 100644 docs/reachability/explainability-schema.md create mode 100644 docs/reachability/graph-revision-schema.md create mode 100644 docs/replay/replay-manifest-v2-acceptance.md create mode 100644 ops/mongo/indices/reachability_store_indices.js create mode 100644 samples/reachability/README.md create mode 100644 samples/reachability/openvex-affected-sample.json create mode 100644 samples/reachability/openvex-not-affected-sample.json create mode 100644 samples/reachability/replay-manifest-v2-sample.json create mode 100644 samples/reachability/richgraph-v1-sample.json create mode 100644 samples/reachability/runtime-facts-sample.ndjson create mode 100644 samples/runtime/java-fat-archive/libs/app-fat.jar create mode 100644 samples/runtime/node-detection-gaps/package.json create mode 100644 samples/runtime/node-detection-gaps/packages/app/package.json create mode 100644 samples/runtime/node-detection-gaps/packages/app/src/index.ts create mode 100644 samples/runtime/node-detection-gaps/packages/app/src/util.ts create mode 100644 samples/runtime/node-detection-gaps/packages/lib/package.json create mode 100644 samples/runtime/node-detection-gaps/packages/lib/src/index.ts create mode 100644 samples/runtime/node-detection-gaps/yarn.lock create mode 100644 src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/MessagingAttestorVerificationCache.cs create mode 100644 src/Authority/StellaOps.Authority/StellaOps.Auth.Client/MessagingTokenCache.cs create mode 100644 src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/Claims/MessagingLdapClaimsCache.cs create mode 100644 src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/NodeBenchMetrics.cs create mode 100644 src/Concelier/StellaOps.Concelier.WebService/Services/MessagingAdvisoryChunkCache.cs create mode 100644 src/Excititor/StellaOps.Excititor.WebService/Services/MessagingGraphOverlayCache.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/Caching/MessagingPolicyEvaluationCache.cs create mode 100644 src/Policy/StellaOps.Policy.Engine/ReachabilityFacts/MessagingReachabilityFactsOverlayCache.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunDeclaredDependencyCollector.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunEvidenceHasher.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockScopeClassifier.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunVersionSpec.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/TASKS.md create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionVfsLoader.cs create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md create mode 100644 src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/Core/LanguageExplicitKey.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/bunfig.toml create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/bun.lock create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/bun.lock create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/bun.lock create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/bun.lock create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.20.patch create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.21.patch create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockScopeClassifierTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/pomxml-only-jar/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/spring-boot-fat-embedded-maven/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/war-embedded-maven/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/layers/layer1/app/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/package-lock.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/pnpm-lock.yaml create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/expected.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/yarn.lock create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/packages/nested/tool/package.json create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeImportWalkerTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeWorkspaceIndexTests.cs create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info/METADATA create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg/__init__.py create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info/METADATA create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg/__init__.py create mode 100644 src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustFixtureBinaries.cs create mode 100644 src/Signals/StellaOps.Signals/Models/ReachabilityStore/CallEdgeDocument.cs create mode 100644 src/Signals/StellaOps.Signals/Models/ReachabilityStore/CveFuncHitDocument.cs create mode 100644 src/Signals/StellaOps.Signals/Models/ReachabilityStore/FuncNodeDocument.cs create mode 100644 src/Signals/StellaOps.Signals/Persistence/IReachabilityStoreRepository.cs create mode 100644 src/Signals/StellaOps.Signals/Persistence/InMemoryReachabilityStoreRepository.cs create mode 100644 src/__Libraries/StellaOps.Auth.Security/Dpop/MessagingDpopReplayCache.cs create mode 100644 src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestV2Tests.cs create mode 100644 src/__Libraries/StellaOps.Replay.Core/CasValidator.cs create mode 100644 src/__Libraries/StellaOps.Replay.Core/ReplayManifestValidator.cs create mode 100644 tests/reachability/StellaOps.Reachability.FixtureTests/FixtureCoverageTests.cs delete mode 100644 tests/reachability/StellaOps.Replay.Core.Tests/ReplayMongoModelsTests.cs delete mode 100644 tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/expect.yaml create mode 100644 tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/ground-truth.json delete mode 100644 tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/expect.yaml create mode 100644 tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/ground-truth.json delete mode 100644 tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/expect.yaml create mode 100644 tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/ground-truth.json delete mode 100644 tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/expect.yaml create mode 100644 tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/ground-truth.json create mode 100644 tests/reachability/fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py create mode 100644 tests/reachability/runners/run_all.ps1 create mode 100644 tests/reachability/runners/run_all.sh diff --git a/= b/= new file mode 100644 index 000000000..e69de29bb diff --git a/Directory.Build.props b/Directory.Build.props index 0b7f9722d..1bf0f7f6a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -24,7 +24,6 @@ - $(PackageTargetFallback);net8.0;net7.0;net6.0;netstandard2.1;netstandard2.0 $(AssetTargetFallback);net8.0;net7.0;net6.0;netstandard2.1;netstandard2.0 diff --git a/bench/README.md b/bench/README.md index be5af2d5e..1aa59a1fa 100644 --- a/bench/README.md +++ b/bench/README.md @@ -1,7 +1,7 @@ -# Stella Ops Bench Repository +# Stella Ops Bench Repository -> **Status:** Draft — aligns with `docs/benchmarks/vex-evidence-playbook.md` (Sprint 401). -> **Purpose:** Host reproducible VEX decisions and comparison data that prove Stella Ops’ signal quality vs. baseline scanners. +> **Status:** Active · Last updated: 2025-12-13 +> **Purpose:** Host reproducible VEX decisions, reachability evidence, and comparison data proving Stella Ops' signal quality vs. baseline scanners. ## Layout @@ -11,20 +11,122 @@ bench/ findings/ # per CVE/product bundles CVE-YYYY-NNNNN/ evidence/ - reachability.json - sbom.cdx.json - decision.openvex.json - decision.dsse.json - rekor.txt - metadata.json + reachability.json # richgraph-v1 excerpt + sbom.cdx.json # CycloneDX SBOM + decision.openvex.json # OpenVEX decision + decision.dsse.json # DSSE envelope + rekor.txt # Rekor log index + inclusion proof + metadata.json # finding metadata (purl, CVE, version) tools/ - verify.sh # DSSE + Rekor verifier + verify.sh # DSSE + Rekor verifier (online) verify.py # offline verifier compare.py # baseline comparison script - replay.sh # runs reachability replay manifolds + replay.sh # runs reachability replay manifests results/ - summary.csv + summary.csv # aggregated metrics runs//... # raw outputs + replay manifests + reachability-benchmark/ # reachability benchmark with JDK fixtures ``` -Refer to `docs/benchmarks/vex-evidence-playbook.md` for artifact contracts and automation tasks. The `bench/` tree will be populated once `BENCH-AUTO-401-019` and `DOCS-VEX-401-012` land. +## Related Documentation + +| Document | Purpose | +|----------|---------| +| [VEX Evidence Playbook](../docs/benchmarks/vex-evidence-playbook.md) | Proof bundle schema, justification catalog, verification workflow | +| [Hybrid Attestation](../docs/reachability/hybrid-attestation.md) | Graph-level and edge-bundle DSSE decisions | +| [Function-Level Evidence](../docs/reachability/function-level-evidence.md) | Cross-module evidence chain guide | +| [Deterministic Replay](../docs/replay/DETERMINISTIC_REPLAY.md) | Replay manifest specification | + +## Verification Workflows + +### Quick Verification (Online) + +```bash +# Verify a VEX proof bundle with DSSE and Rekor +./tools/verify.sh findings/CVE-2021-44228/decision.dsse.json + +# Output: +# ✓ DSSE signature valid +# ✓ Rekor inclusion verified (log index: 12345678) +# ✓ Evidence hashes match +# ✓ Justification catalog membership confirmed +``` + +### Offline Verification + +```bash +# Verify without network access +python tools/verify.py \ + --bundle findings/CVE-2021-44228/decision.dsse.json \ + --cas-root ./findings/CVE-2021-44228/evidence/ \ + --catalog ../docs/benchmarks/vex-justifications.catalog.json + +# Or use the VEX proof bundle verifier +python ../scripts/vex/verify_proof_bundle.py \ + --bundle ../tests/Vex/ProofBundles/sample-proof-bundle.json \ + --cas-root ../tests/Vex/ProofBundles/cas/ +``` + +### Reachability Graph Verification + +```bash +# Verify graph DSSE +stella graph verify --hash blake3:a1b2c3d4... + +# Verify with edge bundles +stella graph verify --hash blake3:a1b2c3d4... --include-bundles + +# Offline with local CAS +stella graph verify --hash blake3:a1b2c3d4... --cas-root ./offline-cas/ +``` + +### Baseline Comparison + +```bash +# Compare Stella Ops findings against baseline scanners +python tools/compare.py \ + --stellaops results/runs/2025-12-13/findings.json \ + --baseline results/baselines/trivy-latest.json \ + --output results/comparison-2025-12-13.csv + +# Metrics generated: +# - True positives (reachability-confirmed) +# - False positives (unreachable code paths) +# - MTTD (mean time to detect) +# - Reproducibility score +``` + +## Artifact Contracts + +All bench artifacts must comply with: + +1. **VEX Proof Bundle Schema** (`docs/benchmarks/vex-evidence-playbook.schema.json`) + - BLAKE3-256 primary hash, SHA-256 secondary + - Canonical JSON with sorted keys + - DSSE envelope with Rekor-ready digest + +2. **Justification Catalog** (`docs/benchmarks/vex-justifications.catalog.json`) + - VEX1-VEX10 justification codes + - Required evidence types per justification + - Expiry and re-evaluation rules + +3. **Reachability Graph** (`docs/contracts/richgraph-v1.md`) + - BLAKE3 graph_hash for content addressing + - Deterministic node/edge ordering + - SymbolID/EdgeID format compliance + +## CI Integration + +The bench directory is validated by: + +- `.gitea/workflows/vex-proof-bundles.yml` - Verifies all proof bundles +- `.gitea/workflows/bench-determinism.yml` - Runs determinism benchmarks +- `.gitea/workflows/hybrid-attestation.yml` - Verifies graph/edge-bundle fixtures + +## Contributing + +1. Add new findings under `findings/CVE-YYYY-NNNNN/` +2. Include all required evidence artifacts +3. Generate DSSE envelope and Rekor proof +4. Update `results/summary.csv` +5. Run verification: `./tools/verify.sh findings/CVE-YYYY-NNNNN/decision.dsse.json` diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index 42355d08c..ce0bf4ca8 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -652,18 +652,95 @@ Signals APIs (base path: `/signals`) provide deterministic ingestion + scoring f | Method | Path | Scope | Notes | |--------|------|-------|-------| -| `POST` | `/signals/callgraphs` | `signals:write` | Ingest a callgraph artifact (base64 JSON); response includes `graphHash` (sha256) and CAS URIs. | -| `POST` | `/signals/runtime-facts` | `signals:write` | Ingest runtime hit events (JSON). | -| `POST` | `/signals/runtime-facts/ndjson` | `signals:write` | Stream NDJSON events (optional gzip) with subject in query params. | +| `POST` | `/signals/callgraphs` | `signals:write` | Ingest a callgraph artifact (richgraph-v1 JSON); response includes `graphHash` (BLAKE3) and CAS URIs. | +| `POST` | `/signals/runtime-facts` | `signals:write` | Ingest runtime hit events (JSON) with `symbolId`, `codeId`, `hitCount`, `loaderBase`. | +| `POST` | `/signals/runtime-facts/ndjson` | `signals:write` | Stream NDJSON events (optional gzip) with `scanId`/`imageDigest` in query params. | | `POST` | `/signals/unknowns` | `signals:write` | Ingest unresolved symbols/edges; influences `unknownsPressure`. | -| `GET` | `/signals/facts/{subjectKey}` | `signals:read` | Fetch `ReachabilityFactDocument` including `metadata.fact.digest` and per-target `states[]`. | +| `GET` | `/signals/facts/{subjectKey}` | `signals:read` | Fetch `ReachabilityFactDocument` including `metadata.fact.digest`, per-target `states[]`, and `latticeState`. | | `POST` | `/signals/reachability/recompute` | `signals:admin` | Recompute reachability for explicit targets and blocked edges. | +**Callgraph ingestion request:** + +```json +{ + "schema": "richgraph-v1", + "analyzer": {"name": "scanner.java", "version": "1.2.0", "toolchain_digest": "sha256:..."}, + "nodes": [ + { + "id": "sym:java:...", + "symbol_id": "sym:java:...", + "code_id": "code:java:...", + "lang": "java", + "kind": "method", + "display": "com.example.Foo.bar()", + "purl": "pkg:maven/com.example/foo@1.0.0", + "symbol_digest": "sha256:...", + "symbol": {"demangled": "com.example.Foo.bar()", "source": "DWARF", "confidence": 0.98} + } + ], + "edges": [{"from": "sym:java:...", "to": "sym:java:...", "kind": "call", "purl": "pkg:maven/...", "symbol_digest": "sha256:...", "confidence": 0.92}], + "roots": [{"id": "sym:java:...", "phase": "runtime", "source": "main"}] +} +``` + +**Callgraph ingestion response:** + +```json +{ + "graphHash": "blake3:a1b2c3d4e5f6...", + "casUri": "cas://reachability/graphs/a1b2c3d4e5f6...", + "dsseUri": "cas://reachability/graphs/a1b2c3d4e5f6....dsse", + "nodeCount": 1247, + "edgeCount": 3891 +} +``` + +**Runtime facts NDJSON fields:** + +| Field | Required | Description | +|-------|----------|-------------| +| `symbolId` | Yes | Canonical `sym:{lang}:{base64url}` | +| `codeId` | No | `code:{lang}:{base64url}` for stripped binaries | +| `hitCount` | No | Number of observed invocations | +| `loaderBase` | No | Memory address base for position-independent code | +| `processId` | No | OS process identifier | +| `containerId` | No | Container runtime identifier | +| `observedAt` | No | ISO-8601 UTC timestamp | + +**Reachability facts response (excerpt):** + +```json +{ + "subjectKey": "scan:123:pkg:maven/log4j:2.14.1:CVE-2021-44228", + "metadata": {"fact": {"digest": "sha256:abc123...", "version": 3}}, + "states": [ + { + "symbol": "sym:java:...", + "latticeState": "CR", + "bucket": "runtime", + "confidence": 0.92, + "score": 0.78, + "path": ["sym:java:main...", "sym:java:log4j..."], + "evidence": { + "static": {"graphHash": "blake3:...", "pathLength": 3}, + "runtime": {"hitCount": 47, "observedAt": "2025-12-13T10:00:00Z"} + } + } + ], + "score": 0.78, + "aggregateTier": "T2", + "riskScore": 0.65 +} +``` + +**Lattice states:** `U` (Unknown), `SR` (StaticallyReachable), `SU` (StaticallyUnreachable), `RO` (RuntimeObserved), `RU` (RuntimeUnobserved), `CR` (ConfirmedReachable), `CU` (ConfirmedUnreachable), `X` (Contested). + Docs & samples: - `docs/api/signals/reachability-contract.md` - `docs/api/signals/samples/callgraph-sample.json` - `docs/api/signals/samples/facts-sample.json` - `docs/reachability/lattice.md` +- `docs/reachability/function-level-evidence.md` ### 2.9 CVSS Receipts (Policy Gateway) @@ -818,6 +895,10 @@ Both commands honour CLI observability hooks: Spectre tables for human output, ` | `stellaops-cli sources ingest --dry-run` | Dry-run guard validation for individual payloads | `--source `
`--input `
`--tenant `
`--format table\|json`
`--output ` | Normalises gzip/base64 payloads, invokes `api/aoc/ingest/dry-run`, and maps guard failures to deterministic `ERR_AOC_00x` exit codes. | | `stellaops-cli aoc verify` | Replay AOC guardrails over stored documents | `--since `
`--limit `
`--sources `
`--codes `
`--format table\|json`
`--export ` | Summarises checked counts/violations, supports JSON evidence exports, and returns `0`, `11…17`, `18`, `70`, or `71` depending on guard outcomes. | | `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | +| `stellaops-cli graph explain` | Show reachability call path for a finding | `--finding ` (required)
`--scan-id `
`--format table\|json` | Displays `latticeState`, call path with `symbol_id`/`code_id`, runtime hits, `graph_hash`, and DSSE attestation refs | +| `stellaops-cli graph export` | Export reachability graph bundle | `--scan-id ` (required)
`--output `
`--include-runtime` | Creates `richgraph-v1.json`, `.dsse`, `meta.json`, and optional `runtime-facts.ndjson` | +| `stellaops-cli graph verify` | Verify graph DSSE signature and Rekor entry | `--graph ` (required)
`--dsse `
`--rekor-log` | Recomputes BLAKE3 hash, validates DSSE envelope, checks Rekor inclusion proof | +| `stellaops-cli replay verify` | Verify replay manifest determinism | `--manifest ` (required)
`--sealed`
`--verbose` | Recomputes all artifact hashes and compares against manifest; exit 0 on match | | `stellaops-cli runtime policy test` | Ask Scanner.WebService for runtime verdicts (Webhook parity) | `--image/-i ` (repeatable, comma/space lists supported)
`--file/-f `
`--namespace/--ns `
`--label/-l key=value` (repeatable)
`--json` | Posts to `POST /api/v1/scanner/policy/runtime`, deduplicates image digests, and prints TTL/policy revision plus per-image columns for signed state, SBOM referrers, quieted-by metadata, confidence, Rekor attestation (uuid + verified flag), and recently observed build IDs (shortened for readability). Accepts newline/whitespace-delimited stdin when piped; `--json` emits the raw response without additional logging. | > Need to debug how the scanner resolves entry points? See the [entry-point documentation index](modules/scanner/operations/entrypoint.md), which links to static/dynamic reducers, ShellFlow, and runtime-specific guides. diff --git a/docs/api/policy.md b/docs/api/policy.md index c4ab4aa23..d0abfd49e 100644 --- a/docs/api/policy.md +++ b/docs/api/policy.md @@ -237,11 +237,83 @@ Slim wrapper used by CLI; returns 204 on success or `ERR_POL_001` payload. Policy Engine evaluations may be enriched with reachability facts produced by Signals. These facts are expected to be: - **Deterministic:** referenced by `metadata.fact.digest` (sha256) and versioned via `metadata.fact.version`. -- **Evidence-linked:** per-target states include `path[]` and `evidence.runtimeHits[]` (and any future CAS/DSSE pointers). +- **Evidence-linked:** per-target states include `path[]`, `evidence.static.graphHash`, `evidence.runtime.hitCount`, and CAS/DSSE pointers. + +#### 6.0.1 Core Identifiers + +| Identifier | Format | Description | +|------------|--------|-------------| +| `symbol_id` | `sym:{lang}:{base64url}` | Canonical function identity (SHA-256 of tuple) | +| `code_id` | `code:{lang}:{base64url}` | Identity for stripped/name-less code blocks | +| `graph_hash` | `blake3:{hex}` | Content-addressable graph identity | +| `fact.digest` | `sha256:{hex}` | Canonical reachability fact digest | + +#### 6.0.2 Lattice States + +Policy gates operate on the 8-state reachability lattice: + +| State | Code | Policy Treatment | +|-------|------|------------------| +| `Unknown` | `U` | Block `not_affected`, allow `under_investigation` | +| `StaticallyReachable` | `SR` | Allow `affected`, block `not_affected` | +| `StaticallyUnreachable` | `SU` | Low-confidence `not_affected` allowed | +| `RuntimeObserved` | `RO` | `affected` required | +| `RuntimeUnobserved` | `RU` | Medium-confidence `not_affected` allowed | +| `ConfirmedReachable` | `CR` | `affected` required, `not_affected` blocked | +| `ConfirmedUnreachable` | `CU` | `not_affected` allowed | +| `Contested` | `X` | `under_investigation` required | + +#### 6.0.3 Evidence Block Schema + +When Policy findings include reachability evidence, the following structure is used: + +```json +{ + "reachability": { + "state": "CR", + "confidence": 0.92, + "evidence": { + "graph_hash": "blake3:a1b2c3d4e5f6...", + "graph_cas_uri": "cas://reachability/graphs/a1b2c3d4e5f6...", + "dsse_uri": "cas://reachability/graphs/a1b2c3d4e5f6....dsse", + "path": [ + {"symbol_id": "sym:java:...", "code_id": "code:java:...", "display": "main()"}, + {"symbol_id": "sym:java:...", "code_id": "code:java:...", "display": "Logger.error()"} + ], + "path_length": 2, + "runtime_hits": 47, + "fact_digest": "sha256:abc123...", + "fact_version": 3 + } + } +} +``` + +#### 6.0.4 Policy Rule Example + +```rego +# Allow not_affected only for confirmed unreachable with high confidence +allow_not_affected { + input.reachability.state == "CU" + input.reachability.confidence >= 0.85 + input.reachability.evidence.fact_digest != "" +} + +# Require affected for confirmed reachable +require_affected { + input.reachability.state == "CR" +} + +# Contested states require investigation +require_investigation { + input.reachability.state == "X" +} +``` Signals contract & scoring model: - `docs/api/signals/reachability-contract.md` - `docs/reachability/lattice.md` +- `docs/reachability/function-level-evidence.md` ### 6.1 Trigger Run diff --git a/docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md b/docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md index f4dde6bd5..07d53aa2c 100644 --- a/docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md +++ b/docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md @@ -51,15 +51,15 @@ | 15 | UI-CLI-401-007 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 1/13/14. | UI & CLI Guilds (`src/Cli/StellaOps.Cli`, `src/UI/StellaOps.UI`) | Implement CLI `stella graph explain` and UI explain drawer with signed call-path, predicates, runtime hits, DSSE pointers, counterfactual controls. | | 16 | QA-DOCS-401-008 | BLOCKED (2025-12-12) | Needs reachbench fixtures (QA-CORPUS-401-031) and docs readiness. | QA & Docs Guilds (`docs`, `tests/README.md`) | Wire reachbench fixtures into CI, document CAS layouts + replay steps, publish operator runbook for runtime ingestion. | | 17 | GAP-SIG-003 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows task 8. | Signals Guild (`src/Signals/StellaOps.Signals`, `docs/reachability/function-level-evidence.md`) | Finish `/signals/runtime-facts` ingestion, add CAS-backed runtime storage, extend scoring to lattice states, emit update events, document retention/RBAC. | -| 18 | SIG-STORE-401-016 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 1/19. | Signals Guild - BE-Base Platform Guild (`src/Signals/StellaOps.Signals`, `src/__Libraries/StellaOps.Replay.Core`) | Introduce shared reachability store collections/indexes and repository APIs for canonical function data. | -| 19 | GAP-REP-004 | BLOCKED (2025-12-13) | Need replay manifest v2 acceptance vectors + CAS registration gates aligned with Signals/Scanner to avoid regressions. | BE-Base Platform Guild (`src/__Libraries/StellaOps.Replay.Core`, `docs/replay/DETERMINISTIC_REPLAY.md`) | Enforce BLAKE3 hashing + CAS registration for graphs/traces, upgrade replay manifest v2, add deterministic tests. | +| 18 | SIG-STORE-401-016 | DONE (2025-12-13) | Complete: added `IReachabilityStoreRepository` + `InMemoryReachabilityStoreRepository` with store models (`FuncNodeDocument`, `CallEdgeDocument`, `CveFuncHitDocument`) and integrated callgraph ingestion to populate the store; Mongo index script at `ops/mongo/indices/reachability_store_indices.js`; Signals test suites passing. | Signals Guild - BE-Base Platform Guild (`src/Signals/StellaOps.Signals`, `src/__Libraries/StellaOps.Replay.Core`) | Introduce shared reachability store collections/indexes and repository APIs for canonical function data. | +| 19 | GAP-REP-004 | DONE (2025-12-13) | Complete: Implemented replay manifest v2 with hash field (algorithm prefix), hashAlg, code_id_coverage, sorted CAS entries. Added ICasValidator interface, ReplayManifestValidator with error codes (REPLAY_MANIFEST_MISSING_VERSION, VERSION_MISMATCH, MISSING_HASH_ALG, UNSORTED_ENTRIES, CAS_NOT_FOUND, HASH_MISMATCH), UpgradeToV2 migration, and 18 deterministic tests per acceptance contract. Files: `ReplayManifest.cs`, `ReachabilityReplayWriter.cs`, `CasValidator.cs`, `ReplayManifestValidator.cs`, `ReplayManifestV2Tests.cs`. | BE-Base Platform Guild (`src/__Libraries/StellaOps.Replay.Core`, `docs/replay/DETERMINISTIC_REPLAY.md`) | Enforce BLAKE3 hashing + CAS registration for graphs/traces, upgrade replay manifest v2, add deterministic tests. | | 20 | GAP-POL-005 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 8/10/17. | Policy Guild (`src/Policy/StellaOps.Policy.Engine`, `docs/modules/policy/architecture.md`, `docs/reachability/function-level-evidence.md`) | Ingest reachability facts into Policy Engine, expose `reachability.state/confidence`, enforce auto-suppress rules, generate OpenVEX evidence blocks. | | 21 | GAP-VEX-006 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows task 20. | Policy, Excititor, UI, CLI & Notify Guilds (`docs/modules/excititor/architecture.md`, `src/Cli/StellaOps.Cli`, `src/UI/StellaOps.UI`, `docs/09_API_CLI_REFERENCE.md`) | Wire VEX emission/explain drawers to show call paths, graph hashes, runtime hits; add CLI flags and Notify templates. | -| 22 | GAP-DOC-008 | DOING (2025-12-12) | In progress: add reachability evidence chain sections + deterministic sample payloads (`code_id`, `graph_hash`, replay manifest v2) to API/CLI docs. | Docs Guild (`docs/reachability/function-level-evidence.md`, `docs/09_API_CLI_REFERENCE.md`, `docs/api/policy.md`) | Publish cross-module function-level evidence guide, update API/CLI references with `code_id`, add OpenVEX/replay samples. | +| 22 | GAP-DOC-008 | DONE (2025-12-13) | Complete: Updated `docs/reachability/function-level-evidence.md` with comprehensive cross-module evidence chain guide (schema, API, CLI, OpenVEX integration, replay manifest v2). Added Signals callgraph/runtime-facts API schema + `stella graph explain/export/verify` CLI commands to `docs/09_API_CLI_REFERENCE.md`. Expanded `docs/api/policy.md` section 6.0 with lattice states, evidence block schema, and Rego policy examples. Created OpenVEX + replay samples under `samples/reachability/` (richgraph-v1-sample.json, openvex-affected/not-affected samples, replay-manifest-v2-sample.json, runtime-facts-sample.ndjson). | Docs Guild (`docs/reachability/function-level-evidence.md`, `docs/09_API_CLI_REFERENCE.md`, `docs/api/policy.md`) | Publish cross-module function-level evidence guide, update API/CLI references with `code_id`, add OpenVEX/replay samples. | | 23 | CLI-VEX-401-011 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 13/14. | CLI Guild (`src/Cli/StellaOps.Cli`, `docs/modules/cli/architecture.md`, `docs/benchmarks/vex-evidence-playbook.md`) | Add `stella decision export|verify|compare`, integrate with Policy/Signer APIs, ship local verifier wrappers for bench artifacts. | | 24 | SIGN-VEX-401-018 | DONE (2025-11-26) | Predicate types added with tests. | Signing Guild (`src/Signer/StellaOps.Signer`, `docs/modules/signer/architecture.md`) | Extend Signer predicate catalog with `stella.ops/vexDecision@v1`, enforce payload policy, plumb DSSE/Rekor integration. | | 25 | BENCH-AUTO-401-019 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 55/58. | Benchmarks Guild (`docs/benchmarks/vex-evidence-playbook.md`, `scripts/bench/**`) | Automate population of `bench/findings/**`, run baseline scanners, compute FP/MTTD/repro metrics, update `results/summary.csv`. | -| 26 | DOCS-VEX-401-012 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows task 22. | Docs Guild (`docs/benchmarks/vex-evidence-playbook.md`, `bench/README.md`) | Maintain VEX Evidence Playbook, publish repo templates/README, document verification workflows. | +| 26 | DOCS-VEX-401-012 | DONE (2025-12-13) | Complete: Updated `bench/README.md` with verification workflows (online/offline/graph), related documentation links, artifact contracts, CI integration, and contributing guidelines. VEX Evidence Playbook already frozen (2025-12-04). | Docs Guild (`docs/benchmarks/vex-evidence-playbook.md`, `bench/README.md`) | Maintain VEX Evidence Playbook, publish repo templates/README, document verification workflows. | | 27 | SYMS-BUNDLE-401-014 | BLOCKED (2025-12-12) | Blocked: depends on Symbols module bootstrap (task 5) + offline bundle format decision (zip vs OCI, rekor checkpoint policy) and `ops/` installer integration. | Symbols Guild - Ops Guild (`src/Symbols/StellaOps.Symbols.Bundle`, `ops`) | Produce deterministic symbol bundles for air-gapped installs with DSSE manifests/Rekor checkpoints; document offline workflows. | | 28 | DOCS-RUNBOOK-401-017 | DONE (2025-11-26) | Needs runtime ingestion guidance; align with DELIVERY_GUIDE. | Docs Guild - Ops Guild (`docs/runbooks/reachability-runtime.md`, `docs/reachability/DELIVERY_GUIDE.md`) | Publish reachability runtime ingestion runbook, link from delivery guides, keep Ops/Signals troubleshooting current. | | 29 | POLICY-LIB-401-001 | DONE (2025-11-27) | Extract DSL parser; align with Policy Engine tasks. | Policy Guild (`src/Policy/StellaOps.PolicyDsl`, `docs/policy/dsl.md`) | Extract policy DSL parser/compiler into `StellaOps.PolicyDsl`, add lightweight syntax, expose `PolicyEngineFactory`/`SignalContext`. | @@ -72,9 +72,9 @@ | 36 | DSSE-DOCS-401-022 | DONE (2025-11-27) | Follows 34/35; document build-time flow. | Docs Guild - Attestor Guild (`docs/ci/dsse-build-flow.md`, `docs/modules/attestor/architecture.md`) | Document build-time attestation walkthrough: models, helper usage, Authority integration, storage conventions, verification commands. | | 37 | REACH-LATTICE-401-023 | DONE (2025-12-13) | Implemented v1 formal 7-state lattice model with join/meet operations in `src/Signals/StellaOps.Signals/Lattice/`. ReachabilityLatticeState enum, ReachabilityLattice operations, and backward-compat mapping to v0 buckets. | Scanner Guild - Policy Guild (`docs/reachability/lattice.md`, `docs/modules/scanner/architecture.md`, `src/Scanner/StellaOps.Scanner.WebService`) | Define reachability lattice model and ensure joins write to event graph schema. | | 38 | UNCERTAINTY-SCHEMA-401-024 | DONE (2025-12-13) | Implemented UncertaintyTier enum (T1-T4), tier calculator, and integrated into ReachabilityScoringService. Documents extended with AggregateTier, RiskScore, and per-state tiers. See `src/Signals/StellaOps.Signals/Lattice/UncertaintyTier.cs`. | Signals Guild (`src/Signals/StellaOps.Signals`, `docs/uncertainty/README.md`) | Extend Signals findings with uncertainty states, entropy fields, `riskScore`; emit update events and persist evidence. | -| 39 | UNCERTAINTY-SCORER-401-025 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows task 38. | Signals Guild (`src/Signals/StellaOps.Signals.Application`, `docs/uncertainty/README.md`) | Implement entropy-aware risk scorer and wire into finding writes. | -| 40 | UNCERTAINTY-POLICY-401-026 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 38/39. | Policy Guild - Concelier Guild (`docs/policy/dsl.md`, `docs/uncertainty/README.md`) | Update policy guidance with uncertainty gates (U1/U2/U3), sample YAML rules, remediation actions. | -| 41 | UNCERTAINTY-UI-401-027 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 38/39. | UI Guild - CLI Guild (`src/UI/StellaOps.UI`, `src/Cli/StellaOps.Cli`, `docs/uncertainty/README.md`) | Surface uncertainty chips/tooltips in Console + CLI output (risk score + entropy states). | +| 39 | UNCERTAINTY-SCORER-401-025 | DONE (2025-12-13) | Complete: reachability risk score now uses configurable entropy weights (`SignalsScoringOptions.UncertaintyEntropyMultiplier` / `UncertaintyBoostCeiling`) and matches `UncertaintyDocument.RiskScore`; added unit coverage in `src/Signals/__Tests/StellaOps.Signals.Tests/ReachabilityScoringServiceTests.cs`. | Signals Guild (`src/Signals/StellaOps.Signals.Application`, `docs/uncertainty/README.md`) | Implement entropy-aware risk scorer and wire into finding writes. | +| 40 | UNCERTAINTY-POLICY-401-026 | DONE (2025-12-13) | Complete: Added uncertainty gates section (§12) to `docs/policy/dsl.md` with U1/U2/U3 gate types, tier-aware compound rules, remediation actions table, and YAML configuration examples. Updated `docs/uncertainty/README.md` with policy guidance (§8) and remediation actions (§9) including CLI commands and automated remediation flow. | Policy Guild - Concelier Guild (`docs/policy/dsl.md`, `docs/uncertainty/README.md`) | Update policy guidance with uncertainty gates (U1/U2/U3), sample YAML rules, remediation actions. | +| 41 | UNCERTAINTY-UI-401-027 | TODO | Unblocked: Tasks 38/39 complete with UncertaintyTier (T1-T4) and entropy-aware scoring. Ready to implement UI/CLI uncertainty display. | UI Guild - CLI Guild (`src/UI/StellaOps.UI`, `src/Cli/StellaOps.Cli`, `docs/uncertainty/README.md`) | Surface uncertainty chips/tooltips in Console + CLI output (risk score + entropy states). | | 42 | PROV-INLINE-401-028 | DONE | Completed inline DSSE hooks per docs. | Authority Guild - Feedser Guild (`docs/provenance/inline-dsse.md`, `src/__Libraries/StellaOps.Provenance.Mongo`) | Extend event writers to attach inline DSSE + Rekor references on every SBOM/VEX/scan event. | | 43 | PROV-BACKFILL-INPUTS-401-029A | DONE | Inventory/map drafted 2025-11-18. | Evidence Locker Guild - Platform Guild (`docs/provenance/inline-dsse.md`) | Attestation inventory and subject->Rekor map drafted. | | 44 | PROV-BACKFILL-401-029 | DONE (2025-11-27) | Use inventory+map; depends on 42/43 readiness. | Platform Guild (`docs/provenance/inline-dsse.md`, `scripts/publish_attestation_with_provenance.sh`) | Resolve historical events and backfill provenance. | @@ -85,21 +85,21 @@ | 49 | GRAPH-PURL-401-034 | DONE (2025-12-11) | purl+symbol_digest in RichGraph nodes/edges (via Sprint 0400 GRAPH-PURL-201-009 + RichGraphBuilder). | Scanner Worker Guild - Signals Guild (`src/Scanner/StellaOps.Scanner.Worker`, `src/Signals/StellaOps.Signals`, `docs/reachability/purl-resolved-edges.md`) | Annotate call edges with callee purl + `symbol_digest`, update schema/CAS, surface in CLI/UI. | | 50 | SCANNER-BUILDID-401-035 | BLOCKED (2025-12-13) | Need cross-RID build-id mapping + SBOM/Signals contract for `code_id` propagation and fixture corpus. | Scanner Worker Guild (`src/Scanner/StellaOps.Scanner.Worker`, `docs/modules/scanner/architecture.md`) | Capture `.note.gnu.build-id` for ELF targets, thread into `SymbolID`/`code_id`, SBOM exports, runtime facts; add fixtures. | | 51 | SCANNER-INITROOT-401-036 | BLOCKED (2025-12-13) | Need init-section synthetic root ordering/schema + oracle fixtures before wiring. | Scanner Worker Guild (`src/Scanner/StellaOps.Scanner.Worker`, `docs/modules/scanner/architecture.md`) | Model init sections as synthetic graph roots (phase=load) including `DT_NEEDED` deps; persist in evidence. | -| 52 | QA-PORACLE-401-037 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 1/53. | QA Guild - Scanner Worker Guild (`tests/reachability`, `docs/reachability/patch-oracles.md`) | Add patch-oracle fixtures and harness comparing graphs vs oracle, fail CI when expected functions/edges missing. | -| 53 | GRAPH-HYBRID-401-053 | BLOCKED (2025-12-13) | Need DSSE/Rekor budget + signing layout decision and golden fixture plan before implementation. | Scanner Worker Guild - Attestor Guild (`src/Scanner/StellaOps.Scanner.Worker`, `src/Attestor/StellaOps.Attestor`, `docs/reachability/hybrid-attestation.md`) | Implement mandatory graph-level DSSE for `richgraph-v1` with deterministic ordering -> BLAKE3 graph hash -> DSSE envelope -> Rekor submit; expose CAS paths `cas://reachability/graphs/{hash}` and `.../{hash}.dsse`; add golden verification fixture. | +| 52 | QA-PORACLE-401-037 | TODO | Unblocked: Tasks 1/53 complete with richgraph-v1 schema and graph-level DSSE. Ready to add patch-oracle fixtures and harness. | QA Guild - Scanner Worker Guild (`tests/reachability`, `docs/reachability/patch-oracles.md`) | Add patch-oracle fixtures and harness comparing graphs vs oracle, fail CI when expected functions/edges missing. | +| 53 | GRAPH-HYBRID-401-053 | DONE (2025-12-13) | Complete: richgraph publisher now stores the canonical `richgraph-v1.json` body at `cas://reachability/graphs/{blake3Hex}` and emits deterministic DSSE envelopes at `cas://reachability/graphs/{blake3Hex}.dsse` (with `DsseCasUri`/`DsseDigest` returned in `RichGraphPublishResult`); added unit coverage validating DSSE payload and signature (`src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/RichGraphPublisherTests.cs`). | Scanner Worker Guild - Attestor Guild (`src/Scanner/StellaOps.Scanner.Worker`, `src/Attestor/StellaOps.Attestor`, `docs/reachability/hybrid-attestation.md`) | Implement mandatory graph-level DSSE for `richgraph-v1` with deterministic ordering -> BLAKE3 graph hash -> DSSE envelope -> Rekor submit; expose CAS paths `cas://reachability/graphs/{hash}` and `.../{hash}.dsse`; add golden verification fixture. | | 54 | EDGE-BUNDLE-401-054 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 51/53. | Scanner Worker Guild - Attestor Guild (`src/Scanner/StellaOps.Scanner.Worker`, `src/Attestor/StellaOps.Attestor`) | Emit optional edge-bundle DSSE envelopes (<=512 edges) for runtime hits, init-array/TLS roots, contested/third-party edges; include `bundle_reason`, per-edge `reason`, `revoked` flag; canonical sort before hashing; Rekor publish capped/configurable; CAS path `cas://reachability/edges/{graph_hash}/{bundle_id}[.dsse]`. | | 55 | SIG-POL-HYBRID-401-055 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows task 54. | Signals Guild - Policy Guild (`src/Signals/StellaOps.Signals`, `src/Policy/StellaOps.Policy.Engine`, `docs/reachability/evidence-schema.md`) | Ingest edge-bundle DSSEs, attach to `graph_hash`, enforce quarantine (`revoked=true`) before scoring, surface presence in APIs/CLI/UI explainers, and add regression tests for graph-only vs graph+bundle paths. | | 56 | DOCS-HYBRID-401-056 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows tasks 53-55. | Docs Guild (`docs/reachability/hybrid-attestation.md`, `docs/modules/scanner/architecture.md`, `docs/modules/policy/architecture.md`, `docs/07_HIGH_LEVEL_ARCHITECTURE.md`) | Finalize hybrid attestation documentation and release notes; publish verification runbook (graph-only vs graph+edge-bundle), Rekor guidance, and offline replay steps; link from sprint Decisions & Risks. | | 57 | BENCH-DETERMINISM-401-057 | DONE (2025-11-26) | Harness + mock scanner shipped; inputs/manifest at `src/Bench/StellaOps.Bench/Determinism/results`. | Bench Guild - Signals Guild - Policy Guild (`bench/determinism`, `docs/benchmarks/signals/`) | Implemented cross-scanner determinism bench (shuffle/canonical), hashes outputs, summary JSON; CI workflow `.gitea/workflows/bench-determinism.yml` runs `scripts/bench/determinism-run.sh`; manifests generated. | | 58 | DATASET-REACH-PUB-401-058 | DONE (2025-12-13) | Test corpus created: JSON schemas at `datasets/reachability/schema/`, 4 samples (csharp/simple-reachable, csharp/dead-code, java/vulnerable-log4j, native/stripped-elf) with ground-truth.json files; test harness at `src/Signals/__Tests/StellaOps.Signals.Tests/GroundTruth/` with 28 validation tests covering lattice states, buckets, uncertainty tiers, gate decisions, path consistency. | QA Guild - Scanner Guild (`tests/reachability/samples-public`, `docs/reachability/evidence-schema.md`) | Materialize PHP/JS/C# mini-app samples + ground-truth JSON (from 23-Nov dataset advisory); runners and confusion-matrix metrics; integrate into CI hot/cold paths with deterministic seeds; keep schema compatible with Signals ingest. | -| 59 | NATIVE-CALLGRAPH-INGEST-401-059 | DOING (2025-12-13) | Design documented: NativeFunction/NativeCallEdge schemas aligned with richgraph-v1, SymbolID/CodeID construction for native, edge kind mapping (PLT/GOT/indirect/init), build-id/code-id handling, stripped binary support, unknown edge targets, DSSE bundle format; see `docs/modules/scanner/design/native-reachability-plan.md` §8. Implementation pending. | Scanner Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Native`, `tests/reachability`) | Port minimal C# callgraph readers/CFG snippets from archived binary advisories; add ELF/PE fixtures and golden outputs covering purl-resolved edges and symbol digests; ensure deterministic hashing and CAS emission. | -| 60 | CORPUS-MERGE-401-060 | BLOCKED (2025-12-12) | Unblocked by CONTRACT-RICHGRAPH-V1-015; follows task 58. | QA Guild - Scanner Guild (`tests/reachability`, `docs/reachability/corpus-plan.md`) | Merge archived multi-runtime corpus (Go/.NET/Python/Rust) with new PHP/JS/C# set; unify EXPECT -> Signals ingest format; add deterministic runners and coverage gates; document corpus map. | +| 59 | NATIVE-CALLGRAPH-INGEST-401-059 | DONE (2025-12-13) | richgraph-v1 alignment tests created at `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Native.Tests/Reachability/RichgraphV1AlignmentTests.cs` with 25 tests validating: SymbolID/EdgeID/RootID/UnknownID formats, SHA-256 digests, deterministic graph hashing, edge type mappings (PLT/InitArray/Indirect), synthetic root phases (load/init/main/fini), stripped binary name format, build-id handling, confidence levels. Fixed pre-existing PeImportParser test bug. | Scanner Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Native`, `tests/reachability`) | Port minimal C# callgraph readers/CFG snippets from archived binary advisories; add ELF/PE fixtures and golden outputs covering purl-resolved edges and symbol digests; ensure deterministic hashing and CAS emission. | +| 60 | CORPUS-MERGE-401-060 | DONE (2025-12-13) | Unblocked: task 58 complete with 4 samples and ground-truth schema. Ready to merge archived multi-runtime corpus. | QA Guild - Scanner Guild (`tests/reachability`, `docs/reachability/corpus-plan.md`) | Merge archived multi-runtime corpus (Go/.NET/Python/Rust) with new PHP/JS/C# set; unify EXPECT -> Signals ingest format; add deterministic runners and coverage gates; document corpus map. | | 61 | DOCS-BENCH-401-061 | DONE (2025-11-26) | Blocks on outputs from 57-60. | Docs Guild (`docs/benchmarks/signals/bench-determinism.md`, `docs/reachability/corpus-plan.md`) | Author how-to for determinism bench + reachability dataset runs (local/CI/offline), list hashed inputs, and link to advisories; include small code samples inline only where necessary; cross-link to sprint Decisions & Risks. | | 62 | VEX-GAPS-401-062 | DONE (2025-12-04) | Schema/catalog frozen; fixtures + verifier landed. | Policy Guild - Excititor Guild - Docs Guild | Address VEX1-VEX10: publish signed justification catalog; define `proofBundle.schema.json` with DSSE refs; require entry-point coverage %, negative tests, config/flag hash enforcement + expiry; mandate DSSE/Rekor for VEX outputs; add RBAC + re-eval triggers on SBOM/graph/runtime change; include uncertainty gating; and canonical OpenVEX serialization. Playbook + schema at `docs/benchmarks/vex-evidence-playbook.{md,schema.json}`; catalog at `docs/benchmarks/vex-justifications.catalog.json` (+ DSSE); fixtures under `tests/Vex/ProofBundles/`; offline verifier `scripts/vex/verify_proof_bundle.py`; CI guard `.gitea/workflows/vex-proof-bundles.yml`. | -| 63 | GRAPHREV-GAPS-401-063 | TODO | None; informs tasks 1, 11, 37-41. | Platform Guild - Scanner Guild - Policy Guild - UI/CLI Guilds | Address graph revision gaps GR1-GR10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: manifest schema + canonical hash rules, mandated BLAKE3-256 encoding, append-only storage, lineage/diff metadata, cross-artifact digests (SBOM/VEX/policy/tool), UI/CLI surfacing of full/short IDs, shard/tenant context, pin/audit governance, retention/tombstones, and inclusion in offline kits. | -| 64 | EXPLAIN-GAPS-401-064 | TODO | None; informs tasks 13-15, 21, 47. | Policy Guild - UI/CLI Guild - Docs Guild - Signals Guild | Address explainability gaps EX1-EX10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: schema/canonicalization + hashes, DSSE predicate/signing policy, CAS storage rules for evidence, link to decision/policy and graph_revision_id, export/replay bundle format, PII/redaction rules, size budgets, versioning, and golden fixtures/tests. | -| 65 | EDGE-GAPS-401-065 | TODO | None; informs tasks 1, 15, 47. | Scanner Guild - Policy Guild - UI/CLI Guild - Docs Guild | Address edge explainability gaps EG1-EG10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: reason enum governance, canonical edge schema with hash rules, evidence limits/redaction, confidence rubric, detector/rule provenance, API/CLI parity, deterministic fixtures, propagation into explanation graphs/VEX, localization guidance, and backfill plan. | -| 66 | BINARY-GAPS-401-066 | TODO | None; informs tasks 12-14, 53-55. | Scanner Guild - Attestor Guild - Policy Guild | Address binary reachability gaps BR1-BR10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: canonical DSSE/predicate schemas, edge hash recipe, required binary evidence with CAS refs, build-id/variant rules, policy hash governance, Sigstore bundle/log routing, idempotent submission keys, size/chunking limits, API/CLI/UI surfacing, and binary fixtures. | +| 63 | GRAPHREV-GAPS-401-063 | DONE (2025-12-13) | Complete: Created `docs/reachability/graph-revision-schema.md` addressing all 10 gaps (GR1-GR10): manifest schema + canonical hash rules, BLAKE3-256 encoding, append-only storage layout, lineage/diff metadata format, cross-artifact digests (SBOM/VEX/policy/tool), UI/CLI full/short ID formats + commands, shard/tenant context, pin/audit governance with events, retention/tombstone policies, and offline kit inclusion with Rekor checkpoints. | Platform Guild - Scanner Guild - Policy Guild - UI/CLI Guilds | Address graph revision gaps GR1-GR10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: manifest schema + canonical hash rules, mandated BLAKE3-256 encoding, append-only storage, lineage/diff metadata, cross-artifact digests (SBOM/VEX/policy/tool), UI/CLI surfacing of full/short IDs, shard/tenant context, pin/audit governance, retention/tombstones, and inclusion in offline kits. | +| 64 | EXPLAIN-GAPS-401-064 | DONE (2025-12-13) | Complete: Created `docs/reachability/explainability-schema.md` addressing all 10 gaps (EX1-EX10): canonical explanation schema + hash rules, DSSE predicate `stella.ops/explanation@v1` + signing policy, CAS storage layout + rules, link format for decision/policy/graph_revision_id, export/replay bundle format with verification, PII/redaction categories + metadata, size budgets with truncation behavior, schema versioning + migration support, golden fixture locations + test categories + CI integration, and determinism guarantees. | Policy Guild - UI/CLI Guild - Docs Guild - Signals Guild | Address explainability gaps EX1-EX10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: schema/canonicalization + hashes, DSSE predicate/signing policy, CAS storage rules for evidence, link to decision/policy and graph_revision_id, export/replay bundle format, PII/redaction rules, size budgets, versioning, and golden fixtures/tests. | +| 65 | EDGE-GAPS-401-065 | DONE (2025-12-13) | Complete: Created `docs/reachability/edge-explainability-schema.md` addressing all 10 gaps (EG1-EG10): reason enum registry with governance rules, canonical edge schema + hash computation using from/to/kind/reason, evidence limits (10 entries) + redaction rules, confidence rubric (certain/high/medium/low/unknown) with base scores per reason, detector/rule provenance schema with input artifact digests, API endpoints + CLI commands with output parity, deterministic fixture locations + requirements, propagation format for explanation graphs + VEX evidence, message catalog structure for localization, and backfill strategy + migration script. | Scanner Guild - Policy Guild - UI/CLI Guild - Docs Guild | Address edge explainability gaps EG1-EG10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: reason enum governance, canonical edge schema with hash rules, evidence limits/redaction, confidence rubric, detector/rule provenance, API/CLI parity, deterministic fixtures, propagation into explanation graphs/VEX, localization guidance, and backfill plan. | +| 66 | BINARY-GAPS-401-066 | DONE (2025-12-13) | Complete: Created `docs/reachability/binary-reachability-schema.md` addressing all 10 gaps (BR1-BR10): canonical DSSE predicates (`stella.ops/binaryGraph@v1`, `stella.ops/binaryEdgeBundle@v1`), edge hash recipe including binary_hash context, required binary evidence table with CAS refs (`cas://binary/blocks|disasm|cfg|symbols`), build-id/variant rules for ELF/PE/Mach-O with fallback, policy hash governance with strict/forward/any binding modes, Sigstore bundle/log routing with offline mode, idempotent submission keys with tenant/binary/graph/hour granularity, size/chunking limits (10MB graph, 512 edges, 1MB DSSE, 100KB Rekor), API endpoints + CLI commands + UI component guidance, and binary fixtures with test categories. | Scanner Guild - Attestor Guild - Policy Guild | Address binary reachability gaps BR1-BR10 from `docs/product-advisories/31-Nov-2025 FINDINGS.md`: canonical DSSE/predicate schemas, edge hash recipe, required binary evidence with CAS refs, build-id/variant rules, policy hash governance, Sigstore bundle/log routing, idempotent submission keys, size/chunking limits, API/CLI/UI surfacing, and binary fixtures. | ## Wave Coordination | Wave | Guild owners | Shared prerequisites | Status | Notes | @@ -127,9 +127,9 @@ ## Action Tracker | # | Action | Owner | Due (UTC) | Status | Notes | | --- | --- | --- | --- | --- | --- | -| 1 | Capture checkpoint dates after Sprint 0400 closure signal. | Planning | 2025-12-15 | TODO | Waiting on Sprint 0400 readiness update. | +| 1 | Capture checkpoint dates after Sprint 0400 closure signal. | Planning | 2025-12-15 | DONE (2025-12-13) | Sprint 0400 archived sprint indicates closed (2025-12-11); checkpoints captured and reflected under Upcoming Checkpoints. | | 2 | Confirm CAS hash alignment (BLAKE3 + sha256 addressing) across Scanner/Replay/Signals. | Platform Guild | 2025-12-10 | DONE (2025-12-10) | CONTRACT-RICHGRAPH-V1-015 adopted; BLAKE3 graph_hash live in Scanner/Replay per GRAPH-CAS-401-001. | -| 3 | Schedule richgraph-v1 schema/hash alignment and rebaseline sprint dates. | Planning - Platform Guild | 2025-12-15 | TODO (slipped) | Rebaseline sprint dates after 2025-12-10 alignment; align with new checkpoints on 2025-12-15/18. | +| 3 | Schedule richgraph-v1 schema/hash alignment and rebaseline sprint dates. | Planning - Platform Guild | 2025-12-15 | DONE (2025-12-12) | Rebaselined checkpoints post 2025-12-10 alignment; updated 2025-12-15/18 readiness reviews (see Execution Log 2025-12-12). | | 4 | Signals ingestion/probe readiness checkpoint for tasks 8-10, 17-18. | Signals Guild - Planning | 2025-12-18 | TODO | Assess runtime ingestion/probe readiness and flip task statuses to DOING/BLOCKED accordingly. | ## Decisions & Risks @@ -153,8 +153,20 @@ ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | -| 2025-12-13 | Documented designs for DOING tasks (37, 38, 48, 58, 59): (1) v1 formal 7-state lattice model with join/meet rules at `docs/reachability/lattice.md` §9; (2) U4 tier and T1-T4 formalized tier definitions at `docs/uncertainty/README.md` §1.1, §5-7; (3) policy gate specification with three gate types at `docs/reachability/policy-gate.md`; (4) ground truth schema for test datasets at `docs/reachability/ground-truth-schema.md`; (5) native callgraph schema alignment with richgraph-v1 at `docs/modules/scanner/design/native-reachability-plan.md` §8. All designs synchronized with existing contracts (richgraph-v1, evidence-schema). Implementation pending for all. | Implementer | -| 2025-12-13 | Marked SCANNER-NATIVE-401-015, GAP-REP-004, SCANNER-BUILDID-401-035, SCANNER-INITROOT-401-036, and GRAPH-HYBRID-401-053 as BLOCKED pending contracts on native lifters/toolchains, replay manifest v2 acceptance vectors/CAS gates, cross-RID build-id/code_id propagation, init synthetic-root schema/oracles, and graph-level DSSE/Rekor budget + golden fixtures. | Planning | +| 2025-12-13 | Unblocked tasks 40/41/52: (1) Task 40 (UNCERTAINTY-POLICY-401-026) now TODO - dependencies 38/39 complete with UncertaintyTier (T1-T4) and entropy-aware scoring. (2) Task 41 (UNCERTAINTY-UI-401-027) now TODO - same dependencies. (3) Task 52 (QA-PORACLE-401-037) now TODO - dependencies 1/53 complete with richgraph-v1 schema and graph-level DSSE. | Implementer | +| 2025-12-13 | Completed CORPUS-MERGE-401-060: migrated `tests/reachability/corpus` from legacy `expect.yaml` to `ground-truth.json` (Reachbench truth schema v1) with updated deterministic manifest generator (`tests/reachability/scripts/update_corpus_manifest.py`) and fixture validation (`tests/reachability/StellaOps.Reachability.FixtureTests/CorpusFixtureTests.cs`). Added cross-dataset coverage gates (`tests/reachability/StellaOps.Reachability.FixtureTests/FixtureCoverageTests.cs`), a deterministic manifest runner for corpus + public samples + reachbench (`tests/reachability/runners/run_all.{sh,ps1}`), and updated corpus map documentation (`docs/reachability/corpus-plan.md`). Fixture tests passing. | Implementer | +| 2025-12-13 | Started CORPUS-MERGE-401-060: unifying `tests/reachability/corpus` and `tests/reachability/samples-public` on a single ground-truth/manifest contract, adding deterministic runners + coverage gates, and updating `docs/reachability/corpus-plan.md`. | Implementer | +| 2025-12-13 | Completed GRAPH-HYBRID-401-053: richgraph CAS publisher now stores canonical JSON bodies and emits deterministic graph DSSE envelopes under `cas://reachability/graphs/{blake3Hex}.dsse`; `RichGraphPublishResult` includes DSSE pointers and tests validate the DSSE payload/signature (`src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityRichGraphPublisher.cs`, `src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/RichGraphPublisherTests.cs`). | Implementer | +| 2025-12-13 | Completed SIG-STORE-401-016 and UNCERTAINTY-SCORER-401-025: added shared reachability store (func_nodes/call_edges/cve_func_hits) repository APIs + Mongo index script (`ops/mongo/indices/reachability_store_indices.js`), integrated store population during callgraph ingestion, and aligned entropy-aware risk scoring so `ReachabilityFactDocument.RiskScore` matches `UncertaintyDocument.RiskScore` with configurable weights; Signals + reachability integration tests passing. | Implementer | +| 2025-12-13 | Started SIG-STORE-401-016 and UNCERTAINTY-SCORER-401-025: implementing reachability store collections/indexes + repository APIs and entropy-aware risk scoring in `src/Signals/StellaOps.Signals`. | Implementer | +| 2025-12-13 | Completed GAP-REP-004: Implemented replay manifest v2 in `src/__Libraries/StellaOps.Replay.Core`. (1) Added `hash` field with algorithm prefix (blake3:..., sha256:...) to ReplayManifest.cs. (2) Added `code_id_coverage` section for stripped binary handling. (3) Created `ICasValidator` interface and `InMemoryCasValidator` for CAS reference validation. (4) Created `ReplayManifestValidator` with error codes per acceptance contract (MISSING_VERSION, VERSION_MISMATCH, MISSING_HASH_ALG, UNSORTED_ENTRIES, CAS_NOT_FOUND, HASH_MISMATCH). (5) Added `UpgradeToV2` migration helper. (6) Added 18 tests covering all v2 acceptance vectors. Also unblocked Task 18 (SIG-STORE-401-016). | Implementer | +| 2025-12-13 | Unblocked tasks 19/26/39/53/60: (1) Created `docs/replay/replay-manifest-v2-acceptance.md` with acceptance vectors, CAS registration gates, test fixtures, and migration path for Task 19. (2) Updated `bench/README.md` with verification workflows, artifact contracts, and CI integration for Task 26 (DONE). (3) Frozen section 8 of `docs/reachability/hybrid-attestation.md` with DSSE/Rekor budget by tier, CAS signing layout, CLI UX, and golden fixture plan for Task 53. (4) Marked Tasks 39 and 60 as TODO since their dependencies (38 and 58) are complete. | Docs Guild | +| 2025-12-13 | Completed BINARY-GAPS-401-066: Created `docs/reachability/binary-reachability-schema.md` addressing all 10 binary reachability gaps (BR1-BR10) from November 2025 product findings. Document specifies: DSSE predicates (`stella.ops/binaryGraph@v1`, `stella.ops/binaryEdgeBundle@v1`), edge hash recipe with binary_hash context, required evidence table with CAS refs, build-id/variant rules for ELF/PE/Mach-O, policy hash governance with binding modes, Sigstore routing with offline mode, idempotent submission keys, size/chunking limits, API/CLI/UI guidance, and binary fixture requirements with test categories. | Docs Guild | +| 2025-12-13 | Completed tasks 37/38/48/58/59: implemented reachability lattice + uncertainty tiers + policy gate evaluator, published ground-truth schema/tests, and added richgraph-v1 native alignment tests; docs synced (`docs/reachability/lattice.md`, `docs/uncertainty/README.md`, `docs/reachability/policy-gate.md`, `docs/reachability/ground-truth-schema.md`, `docs/modules/scanner/design/native-reachability-plan.md`). | Implementer | +| 2025-12-13 | Regenerated deterministic reachbench/corpus manifest hashes with offline scripts (`tests/reachability/fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py`, `tests/reachability/corpus/update_manifest.py`) and verified reachability test suites (Policy Engine, Scanner Reachability, FixtureTests, Signals Reachability, ScannerSignals Integration) passing. | Implementer | +| 2025-12-13 | Closed Action Tracker items #1/#3: captured Sprint 0400 closure signal (archived sprint closed 2025-12-11) and marked richgraph alignment/rebaseline action complete. | Planning | +| 2025-12-13 | Started GAP-REP-004: implementing replay manifest v2 acceptance contract (hash fields, CAS registration gates, deterministic vectors) per `docs/replay/replay-manifest-v2-acceptance.md`. | Implementer | +| 2025-12-13 | Marked SCANNER-NATIVE-401-015, SCANNER-BUILDID-401-035, and SCANNER-INITROOT-401-036 as BLOCKED pending contracts on native lifters/toolchains, cross-RID build-id/code_id propagation, and init synthetic-root schema/oracles. | Planning | | 2025-12-12 | Normalized sprint header/metadata formatting and aligned Action Tracker status labels to `TODO`/`DONE`; no semantic changes. | Project Mgmt | | 2025-12-12 | Rebaselined reachability wave: marked tasks 6/8/13-18/20-21/23/25-26/39-41/46-47/52/54-56/60 as BLOCKED pending upstream deps; set Wave 0401 status to DOING post richgraph alignment so downstream work can queue cleanly. | Planning | | 2025-12-12 | RecordModeService bumped to replay manifest v2 (hashAlg fields, BLAKE3 graph hashes) and ReachabilityReplayWriter now emits hashAlg for graphs/traces; added synthetic runtime probe endpoint to Signals with deterministic builder + tests. | Implementer | diff --git a/docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md b/docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md index 7316088c3..3d255ca3c 100644 --- a/docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md +++ b/docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md @@ -23,30 +23,30 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | SCAN-JAVA-403-001 | TODO | Decide nested locator scheme (Action 1), then implement. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Scan embedded libraries inside archives**: extend `JavaLanguageAnalyzer` to enumerate and parse Maven coordinates from embedded JARs in `BOOT-INF/lib/**.jar`, `WEB-INF/lib/**.jar`, `APP-INF/lib/**.jar`, and `lib/**.jar` *without extracting to disk*. Emit one component per discovered embedded artifact (PURL-based when possible). Evidence locators must represent nesting deterministically (e.g., `outer.jar!BOOT-INF/lib/inner.jar!META-INF/maven/.../pom.properties`). Enforce size/time bounds (skip embedded jars above a configured size threshold; record `embeddedScanSkipped=true` + reason metadata). | -| 2 | SCAN-JAVA-403-002 | TODO | After task 1 skeleton lands, add `pom.xml` fallback and coverage fixtures. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Add `pom.xml` fallback when `pom.properties` is missing**: detect and parse `META-INF/maven/**/pom.xml` (both top-level archives and embedded jars). Prefer `pom.properties` when both exist; otherwise derive `groupId/artifactId/version/packaging/name` from `pom.xml` and emit `pkg:maven/...` PURLs. Evidence must include sha256 of the parsed `pom.xml` entry. If `pom.xml` is present but coordinates are incomplete, emit a component with explicit key (no PURL) carrying `manifestTitle/manifestVersion` and an `unresolvedCoordinates=true` marker (do not guess a Maven PURL). | -| 3 | SCAN-JAVA-403-003 | TODO | Requires agreement on multi-module precedence (Interlock 2). | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Parse all discovered Gradle lockfiles deterministically**: update `JavaLockFileCollector` to parse lockfiles from `JavaBuildFileDiscovery` results (not only root `gradle.lockfile` and `gradle/dependency-locks`). Preserve the lockfile-relative path as `lockLocator` and include module context in metadata (e.g., `lockModulePath`). Deduplicate identical GAVs deterministically (stable overwrite rules documented in code + tested). | -| 4 | SCAN-JAVA-403-004 | TODO | Decide runtime component identity strategy (Action 2). | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Emit runtime image components**: when `JavaWorkspaceNormalizer` identifies a runtime image, emit a `java-runtime` component (explicit key or PURL per decision) with metadata `java.version`, `java.vendor`, and `runtimeImagePath` (relative). Evidence must reference the `release` file. Ensure deterministic ordering and do not double-count multiple identical runtime images (same version+vendor+relative path). | -| 5 | SCAN-JAVA-403-005 | TODO | After task 1 or 2, wire bytecode JNI analysis once per scan. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Replace naive JNI string scanning with bytecode-based JNI analysis**: integrate `Internal/Jni/JavaJniAnalyzer` into `JavaLanguageAnalyzer` so JNI usage metadata is derived from parsed method invocations and native method flags (not raw ASCII search). Output must be bounded and deterministic: emit counts + top-N stable samples (e.g., `jni.edgeCount`, `jni.targetLibraries`, `jni.reasons`). Do not emit full class lists unbounded. | -| 6 | SCAN-JAVA-403-006 | TODO | Parallel with tasks 1–5; keep fixtures minimal. | QA Guild (`src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests`) | **Add fixtures + golden outputs for new detection paths**: introduce fixtures covering (a) fat JAR with embedded libs under `BOOT-INF/lib`, (b) WAR with embedded libs under `WEB-INF/lib`, (c) artifact containing only `pom.xml` (no `pom.properties`), (d) multi-module Gradle lockfile layout, and (e) runtime image directory with `release`. Add/extend `JavaLanguageAnalyzerTests.cs` golden harness assertions proving embedded components are emitted with correct nested locators and stable ordering. | -| 7 | SCAN-JAVA-403-007 | TODO | After tasks 1–2 land, wire perf guard. | Bench Guild (`src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Add benchmark scenario for fat-archive scanning**: add a deterministic bench case that scans a representative fat JAR fixture and reports component count + elapsed time. Establish a baseline ceiling and ensure CI can run it offline. | -| 8 | SCAN-JAVA-403-008 | TODO | After tasks 1–5 land, document final contract. | Docs Guild + Java Analyzer Guild (`docs/modules/scanner`, `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Document Java analyzer detection contract**: update `docs/modules/scanner/architecture.md` (or add a Java analyzer sub-doc under `docs/modules/scanner/`) describing: embedded jar scanning rules, nested evidence locator format, lock precedence rules, runtime component emission, JNI metadata semantics, and known limitations (e.g., shaded jars with stripped Maven metadata remain best-effort). Link this sprint from the doc’s “evidence & determinism” area. | +| 1 | SCAN-JAVA-403-001 | DONE | Embedded scan ships with bounds + nested locators; fixtures/goldens in task 6 validate. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Scan embedded libraries inside archives**: extend `JavaLanguageAnalyzer` to enumerate and parse Maven coordinates from embedded JARs in `BOOT-INF/lib/**.jar`, `WEB-INF/lib/**.jar`, `APP-INF/lib/**.jar`, and `lib/**.jar` *without extracting to disk*. Emit one component per discovered embedded artifact (PURL-based when possible). Evidence locators must represent nesting deterministically (e.g., `outer.jar!BOOT-INF/lib/inner.jar!META-INF/maven/.../pom.properties`). Enforce size/time bounds (skip embedded jars above a configured size threshold; record `embeddedScanSkipped=true` + reason metadata). | +| 2 | SCAN-JAVA-403-002 | DONE | `pom.xml` fallback implemented for archives + embedded jars; explicit-key unresolved when incomplete. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Add `pom.xml` fallback when `pom.properties` is missing**: detect and parse `META-INF/maven/**/pom.xml` (both top-level archives and embedded jars). Prefer `pom.properties` when both exist; otherwise derive `groupId/artifactId/version/packaging/name` from `pom.xml` and emit `pkg:maven/...` PURLs. Evidence must include sha256 of the parsed `pom.xml` entry. If `pom.xml` is present but coordinates are incomplete, emit a component with explicit key (no PURL) carrying `manifestTitle/manifestVersion` and an `unresolvedCoordinates=true` marker (do not guess a Maven PURL). | +| 3 | SCAN-JAVA-403-003 | BLOCKED | Needs an explicit, documented precedence rule for multi-module lock sources (Interlock 2). | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Parse all discovered Gradle lockfiles deterministically**: update `JavaLockFileCollector` to parse lockfiles from `JavaBuildFileDiscovery` results (not only root `gradle.lockfile` and `gradle/dependency-locks`). Preserve the lockfile-relative path as `lockLocator` and include module context in metadata (e.g., `lockModulePath`). Deduplicate identical GAVs deterministically (stable overwrite rules documented in code + tested). | +| 4 | SCAN-JAVA-403-004 | BLOCKED | Needs runtime component identity decision (Action 2) to avoid false vuln matches. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Emit runtime image components**: when `JavaWorkspaceNormalizer` identifies a runtime image, emit a `java-runtime` component (explicit key or PURL per decision) with metadata `java.version`, `java.vendor`, and `runtimeImagePath` (relative). Evidence must reference the `release` file. Ensure deterministic ordering and do not double-count multiple identical runtime images (same version+vendor+relative path). | +| 5 | SCAN-JAVA-403-005 | DONE | Bytecode JNI metadata integrated and bounded; tests updated. | Java Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Replace naive JNI string scanning with bytecode-based JNI analysis**: integrate `Internal/Jni/JavaJniAnalyzer` into `JavaLanguageAnalyzer` so JNI usage metadata is derived from parsed method invocations and native method flags (not raw ASCII search). Output must be bounded and deterministic: emit counts + top-N stable samples (e.g., `jni.edgeCount`, `jni.targetLibraries`, `jni.reasons`). Do not emit full class lists unbounded. | +| 6 | SCAN-JAVA-403-006 | BLOCKED | Embedded/pomxml goldens landed; lock+runtime fixtures await tasks 3/4 decisions. | QA Guild (`src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests`) | **Add fixtures + golden outputs for new detection paths**: introduce fixtures covering (a) fat JAR with embedded libs under `BOOT-INF/lib`, (b) WAR with embedded libs under `WEB-INF/lib`, (c) artifact containing only `pom.xml` (no `pom.properties`), (d) multi-module Gradle lockfile layout, and (e) runtime image directory with `release`. Add/extend `JavaLanguageAnalyzerTests.cs` golden harness assertions proving embedded components are emitted with correct nested locators and stable ordering. | +| 7 | SCAN-JAVA-403-007 | DONE | Added `java_fat_archive` scenario + fixture `samples/runtime/java-fat-archive`; baseline row pending in follow-up. | Bench Guild (`src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Add benchmark scenario for fat-archive scanning**: add a deterministic bench case that scans a representative fat JAR fixture and reports component count + elapsed time. Establish a baseline ceiling and ensure CI can run it offline. | +| 8 | SCAN-JAVA-403-008 | DONE | Added Java analyzer contract doc + linked from scanner architecture; cross-analyzer contract cleaned. | Docs Guild + Java Analyzer Guild (`docs/modules/scanner`, `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java`) | **Document Java analyzer detection contract**: update `docs/modules/scanner/architecture.md` (or add a Java analyzer sub-doc under `docs/modules/scanner/`) describing: embedded jar scanning rules, nested evidence locator format, lock precedence rules, runtime component emission, JNI metadata semantics, and known limitations (e.g., shaded jars with stripped Maven metadata remain best-effort). Link this sprint from the doc's `evidence & determinism` area. | ## Wave Coordination | Wave | Guild owners | Shared prerequisites | Status | Notes | | --- | --- | --- | --- | --- | -| A: Embedded Inventory | Java Analyzer Guild + QA Guild | Locator decision (Action 1) | TODO | Enables detection of fat JAR/WAR embedded libs. | -| B: Coordinates Fallback | Java Analyzer Guild + QA Guild | None | TODO | `pom.xml` fallback for Maven coordinates when properties missing. | -| C: Lock Coverage | Java Analyzer Guild + QA Guild | Precedence decision (Interlock 2) | TODO | Multi-module Gradle lock ingestion improvements. | -| D: Runtime & JNI Context | Java Analyzer Guild + QA Guild | Runtime identity decision (Action 2) | TODO | Runtime component emission + JNI bytecode integration. | -| E: Bench & Docs | Bench Guild + Docs Guild | Waves A–D | TODO | Perf ceiling + contract documentation. | +| A: Embedded Inventory | Java Analyzer Guild + QA Guild | Locator decision (Action 1) | DOING | Enables detection of fat JAR/WAR embedded libs. | +| B: Coordinates Fallback | Java Analyzer Guild + QA Guild | None | DOING | `pom.xml` fallback for Maven coordinates when properties missing. | +| C: Lock Coverage | Java Analyzer Guild + QA Guild | Precedence decision (Interlock 2) | BLOCKED | Multi-module Gradle lock ingestion improvements. | +| D: Runtime & JNI Context | Java Analyzer Guild + QA Guild | Runtime identity decision (Action 2) | DOING | JNI bytecode integration in progress; runtime emission blocked. | +| E: Bench & Docs | Bench Guild + Docs Guild | Waves A-D | TODO | Perf ceiling + contract documentation. | ## Wave Detail Snapshots - **Wave A:** Embedded JAR enumeration + nested evidence locators; fixtures prove fat-archive dependency visibility. -- **Wave B:** `pom.xml` fallback emits Maven PURLs when properties missing; explicit-key “unknown coords” component when insufficient data. +- **Wave B:** `pom.xml` fallback emits Maven PURLs when properties missing; explicit-key `unknown coords` component when insufficient data. - **Wave C:** Broader Gradle lock ingestion across multi-module layouts; deterministic de-dupe rules and module-context metadata. - **Wave D:** Runtime image component emitted from `release`; JNI metadata uses bytecode parsing with bounded output. -- **Wave E:** Offline benchmark + documented “what the analyzer promises” contract. +- **Wave E:** Offline benchmark + documented `what the analyzer promises` contract. ## Interlocks - Evidence locator format must be stable across analyzers and safe for downstream consumers (CLI/UI/export). (Action 1) @@ -64,23 +64,28 @@ ## Action Tracker | # | Action | Owner | Due (UTC) | Status | Notes | | --- | --- | --- | --- | --- | --- | -| 1 | Decide and document nested evidence locator scheme for embedded JAR entries (`outer!inner!path`). | Project Mgmt + Java Analyzer Guild | 2025-12-13 | Open | Must be stable, deterministic, and parseable by exporters. | +| 1 | Decide and document nested evidence locator scheme for embedded JAR entries (`outer!inner!path`). | Project Mgmt + Java Analyzer Guild | 2025-12-13 | Implemented (pending approval) | Implemented via nested `!` locators (consistent with existing `BuildLocator`); covered by new goldens. | | 2 | Decide runtime component identity approach (explicit key vs PURL scheme; if PURL, specify qualifiers). | Project Mgmt + Scanner Guild | 2025-12-13 | Open | Avoid false vuln matches; prefer explicit-key if uncertain. | -| 3 | Define embedded-scan bounds (max embedded jars per archive, max embedded jar size) and required metadata when skipping. | Java Analyzer Guild + Security Guild | 2025-12-13 | Open | Must prevent resource exhaustion from untrusted artifacts. | +| 3 | Define embedded-scan bounds (max embedded jars per archive, max embedded jar size) and required metadata when skipping. | Java Analyzer Guild + Security Guild | 2025-12-13 | DONE | Implemented hard bounds + deterministic skip markers; documented in `docs/modules/scanner/analyzers-java.md`. | ## Decisions & Risks -- **Decision (pending):** Embedded locator format and runtime identity strategy (see Action Tracker 1–2). +- **Decision (pending):** Embedded locator format and runtime identity strategy (see Action Tracker 1-2). + - **Note:** This sprint proceeds using the existing Java analyzer locator convention (`archiveRelativePath!entryPath`), extended by nesting additional `!` separators for embedded jars. + - **Note:** Unresolved `pom.xml` coordinates emit an explicit-key component via `LanguageExplicitKey.Create("java","maven",...)` with `purl=null` and `version=null` (metadata still carries `manifestVersion`). + - **Blockers:** `SCAN-JAVA-403-003` (lock precedence) and `SCAN-JAVA-403-004` (runtime identity). | Risk ID | Risk | Impact | Likelihood | Mitigation | Owner | Trigger / Signal | | --- | --- | --- | --- | --- | --- | --- | | R1 | Embedded jar scanning increases CPU/memory and can be abused by large payloads. | High | Medium | Hard limits + streaming where possible; deterministic skip markers; add perf bench. | Java Analyzer Guild | Bench regression; OOM/timeout in CI; unusually large jar fixtures. | | R2 | Nested locator format breaks downstream tooling expectations (export/UI). | Medium | Medium | Decide format up-front; add tests that assert exact locator strings; document contract. | Project Mgmt | Export bundle consumers fail parsing; UI shows confusing paths. | | R3 | `pom.xml` parsing yields partial/incorrect coordinates (parent inheritance not available). | Medium | Medium | Only emit Maven PURL when `groupId/artifactId/version` are present; otherwise explicit-key component with `unresolvedCoordinates=true`. | Java Analyzer Guild | Golden fixtures show non-deterministic/missing coordinates. | -| R4 | Multi-module lock ingestion causes duplicate “declared-only” components or unstable overwrite rules. | Medium | Medium | Define precedence; stable sort and deterministic overwrite; fixture covering duplicates. | Java Analyzer Guild | Flaky tests; differing outputs depending on directory order. | -| R5 | Runtime “PURL” choice creates false vuln matches for Java runtimes. | High | Low/Medium | Prefer explicit-key component unless a vetted PURL scheme is agreed. | Scanner Guild | Vuln matches spike for runtime-only components without evidence. | +| R4 | Multi-module lock ingestion causes duplicate `declared-only` components or unstable overwrite rules. | Medium | Medium | Define precedence; stable sort and deterministic overwrite; fixture covering duplicates. | Java Analyzer Guild | Flaky tests; differing outputs depending on directory order. | +| R5 | Runtime `PURL` choice creates false vuln matches for Java runtimes. | High | Low/Medium | Prefer explicit-key component unless a vetted PURL scheme is agreed. | Scanner Guild | Vuln matches spike for runtime-only components without evidence. | ## Execution Log | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-12-12 | Sprint created to close Java analyzer detection gaps (embedded libs, `pom.xml` fallback, lock coverage, runtime images, JNI integration) with fixtures/bench/docs expectations. | Project Mgmt | +| 2025-12-13 | Set tasks 1/2/5 to DOING; marked tasks 3/4 BLOCKED pending precedence/runtime identity decisions; started implementation work. | Java Analyzer Guild | +| 2025-12-13 | DONE: embedded jar scan + `pom.xml` fallback + JNI bytecode metadata; added goldens for fat JAR/WAR/pomxml-only; added bench scenario + Java analyzer contract docs; task 6 remains BLOCKED on tasks 3/4. | Java Analyzer Guild | diff --git a/docs/implplan/SPRINT_0405_0001_0001_scanner_python_detection_gaps.md b/docs/implplan/SPRINT_0405_0001_0001_scanner_python_detection_gaps.md index 0b6a3a2c8..cf7cb3b9e 100644 --- a/docs/implplan/SPRINT_0405_0001_0001_scanner_python_detection_gaps.md +++ b/docs/implplan/SPRINT_0405_0001_0001_scanner_python_detection_gaps.md @@ -20,14 +20,14 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | SCAN-PY-405-001 | TODO | Approve identity/precedence rules (Actions 1–2). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Wire layout-aware discovery into `PythonLanguageAnalyzer`**: stop treating “any `*.dist-info` anywhere” as an installed package source. Use `PythonInputNormalizer` + `PythonVirtualFileSystem` + `PythonPackageDiscovery` as the first-pass inventory (site-packages, editable paths, wheels, zipapps, container layer roots). Ensure deterministic path precedence (later/higher-confidence wins) and bounded scanning (no unbounded full-tree recursion for patterns). Emit package-kind + confidence metadata (`pkg.kind`, `pkg.confidence`, `pkg.location`) for every component. | -| 2 | SCAN-PY-405-002 | TODO | After task 1, define dist-info/egg-info enrichment rules. | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Preserve dist-info “deep evidence” while expanding coverage**: for any discovered package with a real `*.dist-info`/`*.egg-info`, continue to enrich with `PythonDistributionLoader` evidence (METADATA/RECORD/WHEEL/entrypoints, RECORD verification stats). For packages discovered without dist-info (e.g., Poetry editable, vendored, zipapp), emit components using `AddFromExplicitKey` with stable identity rules (Action 1) and evidence pointing to the originating file(s) (`pyproject.toml`, lockfile, archive path). | -| 3 | SCAN-PY-405-003 | TODO | Decide lock precedence + supported formats scope (Action 2). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Expand lockfile/requirements detection and parsing**: upgrade `PythonLockFileCollector` to (a) discover lock/requirements files deterministically (root + nested common paths), (b) support `-r/--requirement` includes with cycle detection, (c) correctly handle editable `-e/--editable` lines, (d) parse PEP 508 specifiers (not only `==/===`) and `name @ url` direct references, and (e) include Pipenv `develop` section. Add opt-in support for at least one modern lock (`uv.lock` or `pdm.lock`) with deterministic record ordering and explicit “unsupported line” counters. | -| 4 | SCAN-PY-405-004 | TODO | Requires container overlay decision (Action 3). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Correct container-layer inventory semantics**: when scanning raw OCI layer trees (`layers/`, `.layers/`, `layer*/`), honor whiteouts/overlay ordering so removed packages are not reported. Use/extend `Internal/Packaging/Adapters/ContainerLayerAdapter` semantics as the source of truth for precedence. Emit explicit metadata markers when inventory is partial due to missing overlay context (e.g., `container.overlayIncomplete=true`). | -| 5 | SCAN-PY-405-005 | TODO | Decide representation for vendored deps (Action 4). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Surface vendored (bundled) Python deps**: integrate `VendoredPackageDetector` so known vendoring patterns (`*_vendor`, `third_party`, `requests.packages`, etc.) are detected. Emit either (a) separate “embedded” components with bounded evidence locators (preferred) or (b) a bounded metadata summary on the parent package (`vendored.detected=true`, `vendored.packages`, `vendored.paths`). Never emit unbounded file/module lists; cap to top-N deterministic samples. | -| 6 | SCAN-PY-405-006 | TODO | After task 1–3, decide “used-by-entrypoint” upgrade approach (Interlock 4). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Improve “used by entrypoint” and scope classification**: today `usedByEntrypoint` primarily comes from RECORD/script hints. Extend this by optionally mapping source-tree imports (`PythonImportAnalysis`) and/or runtime evidence (`PythonRuntimeEvidenceCollector`) to packages (via `TopLevelModules`) so “likely used” can be signaled deterministically (bounded, opt-in). Add `scope` metadata using `PythonScopeClassifier` (prod/dev/docs/build) based on lock sections and requirements file names. | -| 7 | SCAN-PY-405-007 | TODO | Parallel with tasks 1–6; fixtures first. | QA Guild (`src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests`) | **Fixtures + golden outputs**: add fixtures proving new detection paths: (a) conda env (`conda-meta/*.json`) without dist-info, (b) requirements with `-r` includes + `-e .` editable, (c) Pipfile.lock with `default` + `develop`, (d) wheel file in workspace (no extraction), (e) zipapp/pyz with embedded requirements, (f) container layers with whiteouts hiding a dist-info dir, (g) vendored dependency directory under a package. Extend `PythonLanguageAnalyzerTests.cs` to assert deterministic ordering, stable identities, and bounded metadata. | -| 8 | SCAN-PY-405-008 | TODO | After core behavior lands, update docs + perf guard. | Docs Guild + Bench Guild (`docs/modules/scanner`, `src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Document + benchmark Python analyzer contract**: update `docs/modules/scanner/architecture.md` (or add a Python analyzer sub-doc) describing detection sources & precedence, lock parsing rules, container overlay semantics, vendoring representation, and identity rules for non-versioned components. Add a deterministic offline bench scanning a representative fixture (many packages + lockfiles) and record baseline ceilings (time + components count). | +| 1 | SCAN-PY-405-001 | DONE | Implement VFS/discovery pipeline; then codify identity/precedence in tests. | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Wire layout-aware discovery into `PythonLanguageAnalyzer`**: stop treating "any `*.dist-info` anywhere" as an installed package source. Use `PythonInputNormalizer` + `PythonVirtualFileSystem` + `PythonPackageDiscovery` as the first-pass inventory (site-packages, editable paths, wheels, zipapps, container layer roots). Ensure deterministic path precedence (later/higher-confidence wins) and bounded scanning (no unbounded full-tree recursion for patterns). Emit package-kind + confidence metadata (`pkg.kind`, `pkg.confidence`, `pkg.location`) for every component. | +| 2 | SCAN-PY-405-002 | BLOCKED | Blocked on Action 1 identity scheme for non-versioned explicit keys. | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Preserve dist-info "deep evidence" while expanding coverage**: for any discovered package with a real `*.dist-info`/`*.egg-info`, continue to enrich with `PythonDistributionLoader` evidence (METADATA/RECORD/WHEEL/entrypoints, RECORD verification stats). For packages discovered without dist-info (e.g., Poetry editable, vendored, zipapp), emit components using `AddFromExplicitKey` with stable identity rules (Action 1) and evidence pointing to the originating file(s) (`pyproject.toml`, lockfile, archive path). | +| 3 | SCAN-PY-405-003 | BLOCKED | Await Action 2 (lock/requirements precedence + supported formats scope). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Expand lockfile/requirements detection and parsing**: upgrade `PythonLockFileCollector` to (a) discover lock/requirements files deterministically (root + nested common paths), (b) support `-r/--requirement` includes with cycle detection, (c) correctly handle editable `-e/--editable` lines, (d) parse PEP 508 specifiers (not only `==/===`) and `name @ url` direct references, and (e) include Pipenv `develop` section. Add opt-in support for at least one modern lock (`uv.lock` or `pdm.lock`) with deterministic record ordering and explicit "unsupported line" counters. | +| 4 | SCAN-PY-405-004 | BLOCKED | Await Action 3 (container overlay handling contract). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Correct container-layer inventory semantics**: when scanning raw OCI layer trees (`layers/`, `.layers/`, `layer*/`), honor whiteouts/overlay ordering so removed packages are not reported. Use/extend `Internal/Packaging/Adapters/ContainerLayerAdapter` semantics as the source of truth for precedence. Emit explicit metadata markers when inventory is partial due to missing overlay context (e.g., `container.overlayIncomplete=true`). | +| 5 | SCAN-PY-405-005 | BLOCKED | Await Action 4 (vendored deps representation contract). | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Surface vendored (bundled) Python deps**: integrate `VendoredPackageDetector` so known vendoring patterns (`*_vendor`, `third_party`, `requests.packages`, etc.) are detected. Emit either (a) separate "embedded" components with bounded evidence locators (preferred) or (b) a bounded metadata summary on the parent package (`vendored.detected=true`, `vendored.packages`, `vendored.paths`). Never emit unbounded file/module lists; cap to top-N deterministic samples. | +| 6 | SCAN-PY-405-006 | BLOCKED | Await Interlock 4 decision on "used-by-entrypoint" semantics. | Python Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python`) | **Improve "used by entrypoint" and scope classification**: today `usedByEntrypoint` primarily comes from RECORD/script hints. Extend this by optionally mapping source-tree imports (`PythonImportAnalysis`) and/or runtime evidence (`PythonRuntimeEvidenceCollector`) to packages (via `TopLevelModules`) so "likely used" can be signaled deterministically (bounded, opt-in). Add `scope` metadata using `PythonScopeClassifier` (prod/dev/docs/build) based on lock sections and requirements file names. | +| 7 | SCAN-PY-405-007 | BLOCKED | Blocked on Actions 2-4 for remaining fixtures (requirements/includes/editables, whiteouts, vendoring). | QA Guild (`src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests`) | **Fixtures + golden outputs**: add fixtures proving new detection paths: (a) conda env (`conda-meta/*.json`) without dist-info, (b) requirements with `-r` includes + `-e .` editable, (c) Pipfile.lock with `default` + `develop`, (d) wheel file in workspace (no extraction), (e) zipapp/pyz with embedded requirements, (f) container layers with whiteouts hiding a dist-info dir, (g) vendored dependency directory under a package. Extend `PythonLanguageAnalyzerTests.cs` to assert deterministic ordering, stable identities, and bounded metadata. | +| 8 | SCAN-PY-405-008 | DONE | After core behavior lands, update docs + perf guard. | Docs Guild + Bench Guild (`docs/modules/scanner`, `src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Document + benchmark Python analyzer contract**: update `docs/modules/scanner/architecture.md` (or add a Python analyzer sub-doc) describing detection sources & precedence, lock parsing rules, container overlay semantics, vendoring representation, and identity rules for non-versioned components. Add a deterministic offline bench scanning a representative fixture (many packages + lockfiles) and record baseline ceilings (time + components count). | ## Wave Coordination | Wave | Guild owners | Shared prerequisites | Status | Notes | @@ -67,7 +67,13 @@ | 4 | Decide how vendored deps are represented (separate embedded components vs parent-only metadata) and how to avoid false vuln matches. | Project Mgmt + Python Analyzer Guild | 2025-12-13 | Open | Prefer separate components only when identity/version is defensible; otherwise bounded metadata summary. | ## Decisions & Risks -- **Decision (pending):** Identity scheme for non-versioned components, lock precedence, and container overlay expectations (Action Tracker 1–3). +- **Decision (pending):** Identity scheme for non-versioned components, lock precedence, and container overlay expectations (Action Tracker 1-3). +- **BLOCKED:** `SCAN-PY-405-002` needs an approved explicit-key identity scheme (Action Tracker 1) before emitting non-versioned components (vendored/local/zipapp/project). +- **BLOCKED:** `SCAN-PY-405-003` awaits lock/requirements precedence + supported formats scope (Action Tracker 2). +- **BLOCKED:** `SCAN-PY-405-004` awaits container overlay handling contract for raw `layers/` inputs (Action Tracker 3). +- **BLOCKED:** `SCAN-PY-405-005` awaits vendored deps representation contract (Action Tracker 4). +- **BLOCKED:** `SCAN-PY-405-006` awaits Interlock 4 decision on "used-by-entrypoint" semantics (avoid turning heuristics into truth). +- **BLOCKED:** `SCAN-PY-405-007` awaits Actions 2-4 to fixture remaining semantics (includes/editables, overlay/whiteouts, vendoring). | Risk ID | Risk | Impact | Likelihood | Mitigation | Owner | Trigger / Signal | | --- | --- | --- | --- | --- | --- | --- | @@ -81,4 +87,12 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-12-12 | Sprint created to close Python analyzer detection gaps (layout-aware discovery, lockfile expansion, container overlay correctness, vendoring signals, optional usage/scope improvements) with fixtures/bench/docs expectations. | Project Mgmt | +| 2025-12-13 | Started SCAN-PY-405-001 (wire VFS/discovery into PythonLanguageAnalyzer). | Python Analyzer Guild | +| 2025-12-13 | Completed SCAN-PY-405-001 (layout-aware VFS-based discovery; pkg.kind/pkg.confidence/pkg.location metadata; deterministic archive roots; updated goldens + tests). | Python Analyzer Guild | +| 2025-12-13 | Started SCAN-PY-405-002 (preserve/enrich dist-info evidence across discovered sources). | Python Analyzer Guild | +| 2025-12-13 | Enforced identity safety for editable lock entries (explicit-key, no `@editable` PURLs, host-path scrubbing) and updated layered fixture to prove `layers/`, `.layers/`, and `layer*/` discovery. | Implementer | +| 2025-12-13 | Added `PythonDistributionVfsLoader` for archive dist-info enrichment (RECORD verification + metadata parity for wheels/zipapps); task remains blocked on explicit-key identity scheme (Action Tracker 1). | Implementer | +| 2025-12-13 | Marked SCAN-PY-405-003 through SCAN-PY-405-007 as `BLOCKED` pending Actions 2-4; synced statuses to `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`. | Implementer | +| 2025-12-13 | Started SCAN-PY-405-008 (document current Python analyzer contract and extend deterministic offline bench coverage). | Implementer | +| 2025-12-13 | Completed SCAN-PY-405-008 (added Python analyzer contract doc + linked from Scanner architecture; extended analyzer microbench config and refreshed baseline; fixed Node analyzer empty-root guard to unblock bench runs from repo root). | Implementer | diff --git a/docs/implplan/SPRINT_0406_0001_0001_scanner_node_detection_gaps.md b/docs/implplan/SPRINT_0406_0001_0001_scanner_node_detection_gaps.md index f9782734a..4414a30ae 100644 --- a/docs/implplan/SPRINT_0406_0001_0001_scanner_node_detection_gaps.md +++ b/docs/implplan/SPRINT_0406_0001_0001_scanner_node_detection_gaps.md @@ -20,17 +20,17 @@ ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | SCAN-NODE-406-001 | TODO | Decide identity/declared-only scheme (Action 1). | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Emit declared-only components**: `NodeLockData.LoadAsync` already builds `DeclaredPackages` from lockfiles + `package.json`, but `NodeLanguageAnalyzer` never emits them. Add a deterministic “declared-only emission” pass that emits components for any `DeclaredPackages` entry not backed by on-disk inventory. Must include: `declaredOnly=true`, `declared.source` (`package.json|package-lock.json|yarn.lock|pnpm-lock.yaml`), `declared.locator` (stable), `declared.versionSpec` (original range/tag), `declared.scope` (prod/dev/peer/optional if known), and `declared.resolvedVersion` (only when lock provides concrete). **Critical:** do not emit `pkg:npm/...@` PURLs; use `AddFromExplicitKey` when version is not a concrete resolved version. | -| 2 | SCAN-NODE-406-002 | TODO | After Action 2, implement + fixtures. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Multi-version lock correctness**: fix `NodeLockData` to support multiple versions per package name and match lock entries by `(name, resolvedVersion)` when the on-disk package.json has a concrete version. Add a `TryGet(relativePath, name, version)` overload (or equivalent) so lock metadata (`integrity`, `resolved`, `scope`) attaches to the correct package instance. Replace/augment `_byName` with a deterministic `(name@version)->entry` map for yarn/pnpm sources. | -| 3 | SCAN-NODE-406-003 | TODO | No external YAML libs; keep deterministic. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Support Yarn Berry (v2/v3) lock format**: extend `NodeLockData.LoadYarnLock` to parse modern `yarn.lock` entries that use `resolution:` / `checksum:` / `linkType:` (and may not have `resolved`/`integrity`). Map `checksum` to an integrity-like field (metadata/evidence) and preserve the raw locator key as `lockLocator`. Ensure multiple versions of the same package are preserved (Task 2). Add fixtures covering Yarn v1 and Yarn v3 lock styles. | -| 4 | SCAN-NODE-406-004 | TODO | Align with Action 3 on “integrity missing” semantics. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Harden pnpm lock parsing**: extend `LoadPnpmLock` to handle packages that have no `integrity` (workspace/file/link/git) without silently dropping them. Emit declared-only entries with `declared.resolvedVersion` (if known) and `lockIntegrityMissing=true` + reason. Add support for newer pnpm layouts (`snapshots:`) when present, while keeping parsing bounded and deterministic. | -| 5 | SCAN-NODE-406-005 | TODO | After task 2, fix path name extraction and tests. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Fix `package-lock.json` nested node_modules naming**: `ExtractNameFromPath` mis-identifies `node_modules/parent/node_modules/child` unless `name` is present. Update extraction to select the last package segment after the last `node_modules` (incl. scoped packages). Add tests that prove nested dependencies are keyed correctly and lock metadata is attached to the right on-disk package. | -| 6 | SCAN-NODE-406-006 | TODO | Decide workspace glob support (Action 2). | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Improve workspace discovery**: `NodeWorkspaceIndex` only supports patterns ending with `/*`. Extend it to support at least `**`-style patterns used in monorepos (e.g., `packages/**`, `apps/*`, `tools/*`). Ensure expansion is deterministic and safe (bounds on directory traversal; ignore `node_modules`). Add fixtures for multi-depth workspace patterns. | -| 7 | SCAN-NODE-406-007 | TODO | After task 6, add scope index for workspaces. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Workspace-aware dependency scopes**: `NodeDependencyIndex` reads only root `package.json`. Extend scope classification to include workspace member manifests so `scope`/`riskLevel` metadata is correct for workspace packages. Must preserve precedence rules (root vs workspace vs lock) and be deterministic. | -| 8 | SCAN-NODE-406-008 | TODO | Requires Action 4 decision on import scanning bounds. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Import scanning correctness + bounds**: `NodeImportWalker` uses `ParseScript` which misses ESM `import` syntax and fails on TS. Improve by attempting `ParseModule` when script parse fails, and add a bounded heuristic fallback for TS (`import ... from`, `export ... from`) when AST parsing fails. Also bound `AttachImports` so it does not recursively scan every file inside `node_modules` trees by default; restrict to source roots/workspace members and/or cap by file count and total bytes, emitting `importScanSkipped=true` + counters when capped. | -| 9 | SCAN-NODE-406-009 | TODO | After task 1, adopt consistent evidence hashing. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Deterministic evidence hashing for on-disk `package.json`**: today tar/zip packages attach `PackageSha256`, but on-disk packages typically do not. Compute sha256 for `package.json` contents for installed packages (bounded: only package.json, not full dir) and attach to root evidence consistently. Do not hash large files; do not add unbounded IO. | -| 10 | SCAN-NODE-406-010 | TODO | Parallel with tasks 1–9; fixtures first. | QA Guild (`src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests`) | **Fixtures + golden outputs**: add/extend fixtures proving: (a) lock-only project (no node_modules) emits declared-only components, (b) Yarn v3 lock parses + multi-version packages preserved, (c) pnpm lock with workspace/link deps doesn’t silently drop, (d) package-lock nested node_modules naming is correct, (e) workspace glob patterns beyond `/*`, (f) container layout where app `package.json` is not at root (e.g., `/app/package.json` inside a layer root) still emits the app component, (g) ESM + TS import scanning captures imports (bounded) and emits deterministic evidence. Update `NodeLanguageAnalyzerTests.cs` and targeted unit tests (`NodeLockDataTests.cs`, `NodePackageCollectorTests.cs`) to assert deterministic ordering and identity rules. | -| 11 | SCAN-NODE-406-011 | TODO | After core behavior lands, update docs + perf guard. | Docs Guild + Bench Guild (`docs/modules/scanner`, `src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Document + benchmark Node analyzer contract**: document precedence (installed vs declared), identity rules for unresolved versions, Yarn/pnpm lock parsing guarantees/limits, workspace discovery rules, import scanning bounds/semantics, and container layout assumptions. Add a deterministic offline bench that scans a representative fixture (workspace + lock-only + import scan enabled) and records elapsed time + component counts (and file-scan counters) with a baseline ceiling. | +| 1 | SCAN-NODE-406-001 | DONE | Emission + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Emit declared-only components**: `NodeLockData.LoadAsync` already builds `DeclaredPackages` from lockfiles + `package.json`, but `NodeLanguageAnalyzer` never emits them. Add a deterministic "declared-only emission" pass that emits components for any `DeclaredPackages` entry not backed by on-disk inventory. Must include: `declaredOnly=true`, `declared.source` (`package.json|package-lock.json|yarn.lock|pnpm-lock.yaml`), `declared.locator` (stable), `declared.versionSpec` (original range/tag), `declared.scope` (prod/dev/peer/optional if known), and `declared.resolvedVersion` (only when lock provides concrete). **Critical:** do not emit `pkg:npm/...@` PURLs; use `AddFromExplicitKey` when version is not a concrete resolved version. | +| 2 | SCAN-NODE-406-002 | DONE | Multi-version matching + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Multi-version lock correctness**: fix `NodeLockData` to support multiple versions per package name and match lock entries by `(name, resolvedVersion)` when the on-disk package.json has a concrete version. Add a `TryGet(relativePath, name, version)` overload (or equivalent) so lock metadata (`integrity`, `resolved`, `scope`) attaches to the correct package instance. Replace/augment `_byName` with a deterministic `(name@version)->entry` map for yarn/pnpm sources. | +| 3 | SCAN-NODE-406-003 | DONE | Yarn Berry parsing + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Support Yarn Berry (v2/v3) lock format**: extend `NodeLockData.LoadYarnLock` to parse modern `yarn.lock` entries that use `resolution:` / `checksum:` / `linkType:` (and may not have `resolved`/`integrity`). Map `checksum` to an integrity-like field (metadata/evidence) and preserve the raw locator key as `lockLocator`. Ensure multiple versions of the same package are preserved (Task 2). Add fixtures covering Yarn v1 and Yarn v3 lock styles. | +| 4 | SCAN-NODE-406-004 | DONE | pnpm hardening + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Harden pnpm lock parsing**: extend `LoadPnpmLock` to handle packages that have no `integrity` (workspace/file/link/git) without silently dropping them. Emit declared-only entries with `declared.resolvedVersion` (if known) and `lockIntegrityMissing=true` + reason. Add support for newer pnpm layouts (`snapshots:`) when present, while keeping parsing bounded and deterministic. | +| 5 | SCAN-NODE-406-005 | DONE | Nested node_modules naming + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Fix `package-lock.json` nested node_modules naming**: `ExtractNameFromPath` mis-identifies `node_modules/parent/node_modules/child` unless `name` is present. Update extraction to select the last package segment after the last `node_modules` (incl. scoped packages). Add tests that prove nested dependencies are keyed correctly and lock metadata is attached to the right on-disk package. | +| 6 | SCAN-NODE-406-006 | DONE | Bounded `*`/`**` workspace expansion + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Improve workspace discovery**: `NodeWorkspaceIndex` only supports patterns ending with `/*`. Extend it to support at least `**`-style patterns used in monorepos (e.g., `packages/**`, `apps/*`, `tools/*`). Ensure expansion is deterministic and safe (bounds on directory traversal; ignore `node_modules`). Add fixtures for multi-depth workspace patterns. | +| 7 | SCAN-NODE-406-007 | DONE | Workspace-aware scopes + tests landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Workspace-aware dependency scopes**: `NodeDependencyIndex` reads only root `package.json`. Extend scope classification to include workspace member manifests so `scope`/`riskLevel` metadata is correct for workspace packages. Must preserve precedence rules (root vs workspace vs lock) and be deterministic. | +| 8 | SCAN-NODE-406-008 | DONE | ESM/TS parsing + bounded import scan landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Import scanning correctness + bounds**: `NodeImportWalker` uses `ParseScript` which misses ESM `import` syntax and fails on TS. Improve by attempting `ParseModule` when script parse fails, and add a bounded heuristic fallback for TS (`import ... from`, `export ... from`) when AST parsing fails. Also bound `AttachImports` so it does not recursively scan every file inside `node_modules` trees by default; restrict to source roots/workspace members and/or cap by file count and total bytes, emitting `importScanSkipped=true` + counters when capped. | +| 9 | SCAN-NODE-406-009 | DONE | On-disk `package.json` hashing + fixtures landed. | Node Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node`) | **Deterministic evidence hashing for on-disk `package.json`**: today tar/zip packages attach `PackageSha256`, but on-disk packages typically do not. Compute sha256 for `package.json` contents for installed packages (bounded: only package.json, not full dir) and attach to root evidence consistently. Do not hash large files; do not add unbounded IO. | +| 10 | SCAN-NODE-406-010 | DONE | Lock-only lockfile fixtures (package-lock/yarn-berry/pnpm) + workspace glob fixture + container app-root discovery; goldens updated. | QA Guild (`src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests`) | **Fixtures + golden outputs**: add/extend fixtures proving: (a) lock-only project (no node_modules) emits declared-only components, (b) Yarn v3 lock parses + multi-version packages preserved, (c) pnpm lock with workspace/link deps doesn’t silently drop, (d) package-lock nested node_modules naming is correct, (e) workspace glob patterns beyond `/*`, (f) container layout where app `package.json` is not at root (e.g., `/app/package.json` inside a layer root) still emits the app component, (g) ESM + TS import scanning captures imports (bounded) and emits deterministic evidence. Update `NodeLanguageAnalyzerTests.cs` and targeted unit tests (`NodeLockDataTests.cs`, `NodePackageCollectorTests.cs`) to assert deterministic ordering and identity rules. | +| 11 | SCAN-NODE-406-011 | DONE | Docs + offline bench scenario (`node_detection_gaps_fixture`) landed; Prom/JSON record import-scan counters. | Docs Guild + Bench Guild (`docs/modules/scanner`, `src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Document + benchmark Node analyzer contract**: document precedence (installed vs declared), identity rules for unresolved versions, Yarn/pnpm lock parsing guarantees/limits, workspace discovery rules, import scanning bounds/semantics, and container layout assumptions. Add a deterministic offline bench that scans a representative fixture (workspace + lock-only + import scan enabled) and records elapsed time + component counts (and file-scan counters) with a baseline ceiling. | ## Wave Coordination | Wave | Guild owners | Shared prerequisites | Status | Notes | @@ -64,10 +64,10 @@ ## Action Tracker | # | Action | Owner | Due (UTC) | Status | Notes | | --- | --- | --- | --- | --- | --- | -| 1 | Decide explicit-key identity scheme for declared-only Node deps (ranges/tags/git/file/workspace) and document it. | Project Mgmt + Scanner Guild | 2025-12-13 | Open | Must not collide with concrete `pkg:npm/...@` PURLs; must be stable across OS paths. | -| 2 | Decide workspace glob expansion rules (supported patterns, bounds, excluded dirs like `node_modules`). | Project Mgmt + Node Analyzer Guild | 2025-12-13 | Open | Keep deterministic and safe under untrusted inputs. | -| 3 | Decide lock metadata precedence when multiple sources exist and when lock lacks integrity/resolution. | Project Mgmt + Node Analyzer Guild | 2025-12-13 | Open | Must be explicit and test-covered; never depend on file traversal order. | -| 4 | Decide import-scanning policy: default enabled/disabled, scope (workspace only vs all packages), and caps to enforce. | Project Mgmt + Node Analyzer Guild | 2025-12-13 | Open | Must prevent runaway scans; skipped scans must be auditable. | +| 1 | Decide explicit-key identity scheme for declared-only Node deps (ranges/tags/git/file/workspace) and document it. | Project Mgmt + Scanner Guild | 2025-12-13 | Done | Implemented via `LanguageExplicitKey` in `docs/modules/scanner/language-analyzers-contract.md`; Node specifics in `docs/modules/scanner/analyzers-node.md`. | +| 2 | Decide workspace glob expansion rules (supported patterns, bounds, excluded dirs like `node_modules`). | Project Mgmt + Node Analyzer Guild | 2025-12-13 | Done | Supports `*` + `**`, skips `node_modules`, bounded traversal; documented in `docs/modules/scanner/analyzers-node.md`. | +| 3 | Decide lock metadata precedence when multiple sources exist and when lock lacks integrity/resolution. | Project Mgmt + Node Analyzer Guild | 2025-12-13 | Done | Precedence: path match > `(name,version)` > name-only; documented in `docs/modules/scanner/analyzers-node.md`. | +| 4 | Decide import-scanning policy: default enabled/disabled, scope (workspace only vs all packages), and caps to enforce. | Project Mgmt + Node Analyzer Guild | 2025-12-13 | Done | Scope: root + workspace members only; caps + skip markers; bench exports `node.importScan.*` metrics (see `docs/modules/scanner/analyzers-node.md`). | ## Decisions & Risks - **Decision (pending):** Declared-only identity scheme, workspace glob bounds, lock precedence, and import scanning caps (Action Tracker 1–4). @@ -84,4 +84,12 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-12-12 | Sprint created to close Node analyzer detection gaps (declared-only emission, multi-version lock fidelity, Yarn Berry/pnpm parsing, workspace glob support, import scanning correctness/bounds, deterministic evidence hashing) with fixtures/bench/docs expectations. | Project Mgmt | +| 2025-12-13 | Completed Wave A/B tasks 406-001..406-005 (declared-only emission, multi-version lock fidelity, Yarn Berry parsing, pnpm integrity-missing + snapshots, nested package-lock naming) with regression tests. | Implementer | +| 2025-12-13 | Completed task 406-006 (bounded `*`/`**` workspace expansion; skips `node_modules`) with unit tests. | Implementer | +| 2025-12-13 | Completed task 406-007 (workspace-aware dependency scopes) with fixture update + tests. | Implementer | +| 2025-12-13 | Completed task 406-008 (ESM/TS import scanning + bounds) with fixture update + tests. | Implementer | +| 2025-12-13 | Completed task 406-009 (on-disk `package.json` sha256 evidence) with fixture updates. | Implementer | +| 2025-12-13 | Updated declared-only emission to use the cross-analyzer explicit-key format and expanded fixtures for `layers/`, `.layers/`, and `layer*/` discovery. | Implementer | +| 2025-12-13 | Completed task 406-010 (fixtures + goldens: lock-only package-lock/yarn-berry/pnpm, workspace globs, container app-root discovery) with regression tests. | Implementer | +| 2025-12-13 | Completed task 406-011 (docs + offline bench: `docs/modules/scanner/analyzers-node.md`, scenario `node_detection_gaps_fixture`, import-scan metrics) with bench/test coverage. | Implementer | diff --git a/docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md b/docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md index 439a01ce7..a9cd2f1a3 100644 --- a/docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md +++ b/docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md @@ -20,20 +20,20 @@ - `docs/modules/scanner/architecture.md` - `src/Scanner/AGENTS.md` - `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/AGENTS.md` -- **Missing today (must be created before tasks flip to DOING):** `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` +- `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` (created 2025-12-13) ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | SCAN-BUN-407-001 | TODO | Decide container root discovery contract (Action 2). | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Container-layer aware project discovery**: extend `Internal/BunProjectDiscoverer.cs` to discover Bun project roots not only under `context.RootPath`, but also under common OCI unpack layouts used elsewhere in scanner: `layers/*`, `.layers/*`, and `layer*` direct children. Do not skip hidden roots wholesale: `.layers` must be included. Keep traversal bounded and deterministic: (a) stable ordering of enumerated directories, (b) explicit depth caps per root, (c) hard cap on total discovered roots, (d) must never recurse into `node_modules/` and must skip large/non-project dirs deterministically. Acceptance: new fixture `lang/bun/container-layers` proves a Bun project placed under `.layers/layer0/app` is found and scanned. | -| 2 | SCAN-BUN-407-002 | TODO | Decide identity rules for non-concrete versions (Action 1). | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Declared-only fallback for bun markers**: if `BunProjectDiscoverer` identifies a project root (via `bunfig.toml`/`package.json`/etc.) but `BunInputNormalizer` returns `None` (no `node_modules`, no `bun.lock`), emit declared-only components from `package.json` dependencies. Requirements: (a) do not emit `pkg:npm/...@` PURLs for version ranges/tags; use `AddFromExplicitKey` when version is not a concrete resolved version, (b) include deterministic metadata `declaredOnly=true`, `declared.source=package.json`, `declared.locator=#
`, `declared.versionSpec=`, `declared.scope=`, and (c) include root package.json evidence with sha256 (bounded). Acceptance: new fixture `lang/bun/bunfig-only` emits declared-only components for both `dependencies` and `devDependencies` with safe identities. | -| 3 | SCAN-BUN-407-003 | TODO | Decide dev/optional/peer semantics for bun.lock v1 (Action 3). | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **bun.lock v1 graph enrichment (dev/optional/peer + edges)**: upgrade `Internal/BunLockParser.cs` to preserve dependency edges from bun.lock v1 array form (capture dependency value/specifier, not only names) and to parse optional peer information when present. Build a bounded dependency graph that starts from root `package.json` declarations (prod/dev/optional/peer) and propagates reachability to lock entries, marking `BunLockEntry.IsDev/IsOptional/IsPeer` deterministically. If the graph cannot disambiguate (multiple versions/specifier mismatch), do not guess; emit `scopeUnknown=true` and keep `IsDev=false` unless positively proven. Acceptance: add fixture `lang/bun/lockfile-dev-classification` demonstrating: (a) dev-only packages are tagged `dev=true` and are excluded when `includeDev=false`, (b) prod packages remain untagged, (c) the decision is stable across OS/filesystem ordering. | -| 4 | SCAN-BUN-407-004 | TODO | After task 3 lands, wire filter & metadata into emission. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Make `includeDev` meaningful**: `Internal/BunLockInventory.cs` currently filters by `entry.IsDev`, but bun.lock array parsing sets `IsDev=false` always. After graph enrichment (Task 3), implement deterministic filtering for lockfile-only scans and ensure installed scans also carry dev/optional/peer metadata when lock data is present. Acceptance: tests show dev filtering affects output only when the analyzer can prove dev reachability; otherwise outputs remain but are marked `scopeUnknown=true`. | -| 5 | SCAN-BUN-407-005 | TODO | Decide patch-keying and path normalization (Action 4). | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Version-specific patch mapping + no absolute paths**: fix `Internal/BunWorkspaceHelper.cs` so `patchedDependencies` keys preserve version specificity (`name@version`), and patch-directory discovery emits **relative** deterministic paths (relative to project root) rather than absolute OS paths. Update `BunLanguageAnalyzer` patch application so it first matches `name@version`, then falls back to `name` only when unambiguous. Acceptance: add fixture `lang/bun/patched-multi-version` with two patch files for the same package name at different versions; output marks only the correct version as patched and never includes absolute paths. | -| 6 | SCAN-BUN-407-006 | TODO | Align locator conventions with Node analyzer (Action 2). | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Evidence strengthening + locator precision**: improve `Internal/BunPackage.CreateEvidence()` so evidence locators are stable and specific: (a) package.json evidence includes sha256 (bounded; if skipped, emit `packageJsonHashSkipped=true` with reason), (b) bun.lock evidence uses locator `bun.lock#packages/` (or another agreed deterministic locator format) instead of plain `bun.lock`, (c) optionally include lockfile sha256 once per project root in a synthetic “bun.lock evidence record” component or via repeated evidence with identical sha256 (bounded). Acceptance: update existing Bun fixtures’ goldens to reflect deterministic hashing and locator formats, with no nondeterministic absolute paths. | -| 7 | SCAN-BUN-407-007 | TODO | Decide identity rules for non-npm sources (Action 1). | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Identity safety for non-npm sources**: `Internal/BunPackage.BuildPurl()` always emits `pkg:npm/@`. Define and implement rules for `SourceType != npm` (git/file/link/workspace/tarball/custom-registry): when `version` is not a concrete registry version, emit `AddFromExplicitKey` (no PURL) and preserve the original specifier/resolved URL in metadata. If a PURL is emitted, it must be valid and must not embed raw specifiers like `workspace:*` as a “version”. Acceptance: add fixture `lang/bun/non-concrete-versions` demonstrating safe identities for `workspace:*` / `link:` / `file:` styles (if representable in bun.lock), with deterministic explicit keys and clear metadata markers. | -| 8 | SCAN-BUN-407-008 | TODO | After tasks 1–7, document analyzer contract. | Docs Guild + Bun Analyzer Guild | **Document Bun analyzer detection contract**: add/update `docs/modules/scanner/analyzers-bun.md` (or the closest existing scanner doc) describing: what artifacts are used (node_modules, bun.lock, package.json), precedence rules, identity rules (PURL vs explicit-key), dev/optional/peer semantics, container layer root handling, and bounds (depth/roots/files/hash limits). Link this sprint from the doc and add a brief “known limitations” section (e.g., bun.lockb unsupported). | -| 9 | SCAN-BUN-407-009 | TODO | Optional; only if perf regression risk materializes. | Bench Guild (`src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Offline benchmark**: add a deterministic bench that scans a representative Bun monorepo fixture (workspaces + many packages) and records elapsed time + component counts. Establish a ceiling and guard against regressions. | +| 1 | SCAN-BUN-407-001 | DONE | Fixture `lang/bun/container-layers` + determinism test passing. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Container-layer aware project discovery**: extend `Internal/BunProjectDiscoverer.cs` to discover Bun project roots not only under `context.RootPath`, but also under common OCI unpack layouts used elsewhere in scanner: `layers/*`, `.layers/*`, and `layer*` direct children. Do not skip hidden roots wholesale: `.layers` must be included. Keep traversal bounded and deterministic: (a) stable ordering of enumerated directories, (b) explicit depth caps per root, (c) hard cap on total discovered roots, (d) must never recurse into `node_modules/` and must skip large/non-project dirs deterministically. Acceptance: new fixture `lang/bun/container-layers` proves a Bun project placed under `.layers/layer0/app` is found and scanned. | +| 2 | SCAN-BUN-407-002 | DONE | Fixture `lang/bun/bunfig-only` + determinism test passing. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Declared-only fallback for bun markers**: if `BunProjectDiscoverer` identifies a project root (via `bunfig.toml`/`package.json`/etc.) but `BunInputNormalizer` returns `None` (no `node_modules`, no `bun.lock`), emit declared-only components from `package.json` dependencies. Requirements: (a) do not emit `pkg:npm/...@` PURLs for version ranges/tags; use `AddFromExplicitKey` when version is not a concrete resolved version, (b) include deterministic metadata `declaredOnly=true`, `declared.source=package.json`, `declared.locator=#
`, `declared.versionSpec=`, `declared.scope=`, and (c) include root package.json evidence with sha256 (bounded). Acceptance: new fixture `lang/bun/bunfig-only` emits declared-only components for both `dependencies` and `devDependencies` with safe identities. | +| 3 | SCAN-BUN-407-003 | DONE | Fixture `lang/bun/lockfile-dev-classification` passing. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **bun.lock v1 graph enrichment (dev/optional/peer + edges)**: upgrade `Internal/BunLockParser.cs` to preserve dependency edges from bun.lock v1 array form (capture dependency value/specifier, not only names) and to parse optional peer information when present. Build a bounded dependency graph that starts from root `package.json` declarations (prod/dev/optional/peer) and propagates reachability to lock entries, marking `BunLockEntry.IsDev/IsOptional/IsPeer` deterministically. If the graph cannot disambiguate (multiple versions/specifier mismatch), do not guess; emit `scopeUnknown=true` and keep `IsDev=false` unless positively proven. Acceptance: add fixture `lang/bun/lockfile-dev-classification` demonstrating: (a) dev-only packages are tagged `dev=true` and are excluded when `includeDev=false`, (b) prod packages remain untagged, (c) the decision is stable across OS/filesystem ordering. | +| 4 | SCAN-BUN-407-004 | DONE | Dev filter verified via fixture goldens. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Make `includeDev` meaningful**: `Internal/BunLockInventory.cs` currently filters by `entry.IsDev`, but bun.lock array parsing sets `IsDev=false` always. After graph enrichment (Task 3), implement deterministic filtering for lockfile-only scans and ensure installed scans also carry dev/optional/peer metadata when lock data is present. Acceptance: tests show dev filtering affects output only when the analyzer can prove dev reachability; otherwise outputs remain but are marked `scopeUnknown=true`. | +| 5 | SCAN-BUN-407-005 | DONE | Fixture `lang/bun/patched-multi-version` passing. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Version-specific patch mapping + no absolute paths**: fix `Internal/BunWorkspaceHelper.cs` so `patchedDependencies` keys preserve version specificity (`name@version`), and patch-directory discovery emits **relative** deterministic paths (relative to project root) rather than absolute OS paths. Update `BunLanguageAnalyzer` patch application so it first matches `name@version`, then falls back to `name` only when unambiguous. Acceptance: add fixture `lang/bun/patched-multi-version` with two patch files for the same package name at different versions; output marks only the correct version as patched and never includes absolute paths. | +| 6 | SCAN-BUN-407-006 | DONE | Goldens updated; bounded sha256 + lock locators added. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Evidence strengthening + locator precision**: improve `Internal/BunPackage.CreateEvidence()` so evidence locators are stable and specific: (a) package.json evidence includes sha256 (bounded; if skipped, emit `packageJson.hashSkipped=true` + `packageJson.hashSkipReason=<...>`), (b) bun.lock evidence uses locator `:packages[@]` instead of plain `bun.lock`, (c) include lockfile sha256 once per project root via repeated evidence sha256 (bounded). Acceptance: update existing Bun fixtures’ goldens to reflect deterministic hashing and locator formats, with no nondeterministic absolute paths. | +| 7 | SCAN-BUN-407-007 | DONE | Fixture `lang/bun/non-concrete-versions` passing. | Bun Analyzer Guild (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun`) | **Identity safety for non-npm sources**: `Internal/BunPackage.BuildPurl()` always emits `pkg:npm/@`. Define and implement rules for `SourceType != npm` (git/file/link/workspace/tarball/custom-registry): when `version` is not a concrete registry version, emit `AddFromExplicitKey` (no PURL) and preserve the original specifier/resolved URL in metadata. If a PURL is emitted, it must be valid and must not embed raw specifiers like `workspace:*` as a “version”. Acceptance: add fixture `lang/bun/non-concrete-versions` demonstrating safe identities for `workspace:*` / `link:` / `file:` styles (if representable in bun.lock), with deterministic explicit keys and clear metadata markers. | +| 8 | SCAN-BUN-407-008 | DONE | Doc `docs/modules/scanner/analyzers-bun.md` published and sprint linked. | Docs Guild + Bun Analyzer Guild | **Document Bun analyzer detection contract**: add/update `docs/modules/scanner/analyzers-bun.md` (or the closest existing scanner doc) describing: what artifacts are used (node_modules, bun.lock, package.json), precedence rules, identity rules (PURL vs explicit-key), dev/optional/peer semantics, container layer root handling, and bounds (depth/roots/files/hash limits). Link this sprint from the doc and add a brief “known limitations” section (e.g., bun.lockb unsupported). | +| 9 | SCAN-BUN-407-009 | DONE | Added scenario `bun_multi_workspace_fixture` in analyzer microbench harness. | Bench Guild (`src/Bench/StellaOps.Bench/Scanner.Analyzers`) | **Offline benchmark**: add a deterministic bench that scans a representative Bun monorepo fixture (workspaces + many packages) and records elapsed time + component counts. Establish a ceiling and guard against regressions. | ## Wave Coordination | Wave | Guild owners | Shared prerequisites | Status | Notes | @@ -67,14 +67,14 @@ ## Action Tracker | # | Action | Owner | Due (UTC) | Status | Notes | | --- | --- | --- | --- | --- | --- | -| 1 | Decide explicit-key identity scheme for Bun declared-only and non-npm sources (ranges/tags/git/file/link/workspace). | Project Mgmt + Scanner Guild | 2025-12-13 | Open | Must not collide with concrete `pkg:npm/...@` identities; must be stable across OS paths. | -| 2 | Decide and document container layer root discovery rules for Bun analyzer (parity with Node’s `layers/.layers/layer*` conventions, depth/roots bounds). | Project Mgmt + Bun Analyzer Guild | 2025-12-13 | Open | Must prevent runaway scans on untrusted rootfs layouts; must be fixture-tested. | -| 3 | Decide bun.lock v1 scope derivation rules (dev/optional/peer) and how uncertainty is represented (`scopeUnknown` markers). | Project Mgmt + Bun Analyzer Guild | 2025-12-13 | Open | Must be deterministic; avoid false “dev=false” claims when graph is ambiguous. | -| 4 | Decide patched dependency keying and deterministic path normalization (relative path base, name@version precedence, fallback rules). | Project Mgmt + Bun Analyzer Guild + Security Guild | 2025-12-13 | Open | Must avoid absolute path leakage; ensure correct version-specific patch attribution. | -| 5 | Create missing module charter: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md`. | Project Mgmt | 2025-12-13 | Open | Required before implementation tasks can enter DOING per global charter. | +| 1 | Decide explicit-key identity scheme for Bun declared-only and non-npm sources (ranges/tags/git/file/link/workspace). | Project Mgmt + Scanner Guild | 2025-12-13 | Done | Implemented per `docs/modules/scanner/language-analyzers-contract.md`. | +| 2 | Decide and document container layer root discovery rules for Bun analyzer (parity with Node's `layers/.layers/layer*` conventions, depth/roots bounds). | Project Mgmt + Bun Analyzer Guild | 2025-12-13 | Done | Implemented per `docs/modules/scanner/language-analyzers-contract.md`; validated by fixture `lang/bun/container-layers`. | +| 3 | Decide bun.lock v1 scope derivation rules (dev/optional/peer) and how uncertainty is represented (`scopeUnknown` markers). | Project Mgmt + Bun Analyzer Guild | 2025-12-13 | Done | Implemented in `Internal/BunLockScopeClassifier.cs` with `scopeUnknown=true` for ambiguity. | +| 4 | Decide patched dependency keying and deterministic path normalization (relative path base, name@version precedence, fallback rules). | Project Mgmt + Bun Analyzer Guild + Security Guild | 2025-12-13 | Done | Implemented in `Internal/BunWorkspaceHelper.cs` (version-specific keys; project-relative patch paths). | +| 5 | Create missing module charter: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md`. | Project Mgmt | 2025-12-13 | Done | Created `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md`. | ## Decisions & Risks -- **Decision (pending):** identity scheme, container discovery, scope derivation, patch rules (Action Tracker 1–4). +- Decisions implemented per `docs/modules/scanner/language-analyzers-contract.md` and documented in `docs/modules/scanner/analyzers-bun.md`. | Risk ID | Risk | Impact | Likelihood | Mitigation | Owner | Trigger / Signal | | --- | --- | --- | --- | --- | --- | --- | @@ -88,4 +88,7 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-12-12 | Sprint created to close Bun analyzer detection gaps (container-layer discovery, declared-only fallback, bun.lock scope graph, version-specific patches, evidence hashing, identity safety) with fixtures/docs/bench expectations. | Project Mgmt | +| 2025-12-13 | Completed SCAN-BUN-407-001 and SCAN-BUN-407-002 with new fixtures (`lang/bun/container-layers`, `lang/bun/bunfig-only`) and deterministic goldens; aligned explicit-key behavior with `docs/modules/scanner/language-analyzers-contract.md`. | Bun Analyzer Guild | +| 2025-12-13 | Completed SCAN-BUN-407-003 through SCAN-BUN-407-008 (scope graph + dev filtering, version-specific patch mapping, bounded sha256 evidence, non-concrete identity safety, and Bun analyzer contract doc). | Bun Analyzer Guild | +| 2025-12-13 | Completed SCAN-BUN-407-009 by wiring the Bun analyzer into the scanner analyzer microbench harness and adding scenario `bun_multi_workspace_fixture`. | Bench Guild | diff --git a/docs/implplan/SPRINT_0408_0001_0001_scanner_language_detection_gaps_program.md b/docs/implplan/SPRINT_0408_0001_0001_scanner_language_detection_gaps_program.md index 8e722b704..51e873ccd 100644 --- a/docs/implplan/SPRINT_0408_0001_0001_scanner_language_detection_gaps_program.md +++ b/docs/implplan/SPRINT_0408_0001_0001_scanner_language_detection_gaps_program.md @@ -27,15 +27,15 @@ - .NET: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.DotNet/AGENTS.md` - Python: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/AGENTS.md` - Node: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/AGENTS.md` - - **Missing today:** `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` (Action 4) + - Bun: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` (created 2025-12-13; Action 4) ## Delivery Tracker | # | Task ID | Status | Key dependency / next step | Owners | Task Definition | | --- | --- | --- | --- | --- | --- | -| 1 | SCAN-PROG-408-001 | TODO | Requires Action 1. | Scanner Guild + Security Guild + Export/UI/CLI Consumers | **Freeze cross-analyzer identity safety contract**: define a single, documented rule-set for when an analyzer emits (a) a concrete PURL and (b) an explicit-key component. Must cover: version ranges/tags, local paths, workspace/link/file deps, git deps, and “unknown” versions. Output: a canonical doc under `docs/modules/scanner/` (path chosen in Action 1) + per-analyzer unit tests asserting “no invalid PURLs” for declared-only / non-concrete inputs. | -| 2 | SCAN-PROG-408-002 | TODO | Requires Action 2. | Scanner Guild + Export/UI/CLI Consumers | **Freeze cross-analyzer evidence locator contract**: define deterministic locator formats for (a) lockfile entries, (b) nested artifacts (e.g., Java “outer!inner!path”), and (c) derived evidence records. Output: canonical doc + at least one golden fixture per analyzer asserting exact locator strings and bounded evidence sizes. | -| 3 | SCAN-PROG-408-003 | TODO | Requires Action 3. | Scanner Guild | **Freeze container layout discovery contract**: define which analyzers must discover projects under `layers/`, `.layers/`, and `layer*/` layouts, how ordering/whiteouts are handled (where applicable), and bounds (depth/roots/files). Output: canonical doc + fixtures proving parity for Node/Bun/Python (and any Java/.NET container behaviors where relevant). | -| 4 | SCAN-PROG-408-004 | TODO | None. | Project Mgmt + Scanner Guild | **Create missing Bun analyzer charter**: add `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` synthesizing constraints from `docs/modules/scanner/architecture.md` and this sprint + `SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md`. Must include: allowed directories, test strategy, determinism rules, identity/evidence conventions, and “no absolute paths” requirement. | +| 1 | SCAN-PROG-408-001 | DOING | Requires Action 1. | Scanner Guild + Security Guild + Export/UI/CLI Consumers | **Freeze cross-analyzer identity safety contract**: define a single, documented rule-set for when an analyzer emits (a) a concrete PURL and (b) an explicit-key component. Must cover: version ranges/tags, local paths, workspace/link/file deps, git deps, and "unknown" versions. Output: a canonical doc under `docs/modules/scanner/` (path chosen in Action 1) + per-analyzer unit tests asserting "no invalid PURLs" for declared-only / non-concrete inputs. | +| 2 | SCAN-PROG-408-002 | DOING | Requires Action 2. | Scanner Guild + Export/UI/CLI Consumers | **Freeze cross-analyzer evidence locator contract**: define deterministic locator formats for (a) lockfile entries, (b) nested artifacts (e.g., Java "outer!inner!path"), and (c) derived evidence records. Output: canonical doc + at least one golden fixture per analyzer asserting exact locator strings and bounded evidence sizes. | +| 3 | SCAN-PROG-408-003 | DOING | Requires Action 3. | Scanner Guild | **Freeze container layout discovery contract**: define which analyzers must discover projects under `layers/`, `.layers/`, and `layer*/` layouts, how ordering/whiteouts are handled (where applicable), and bounds (depth/roots/files). Output: canonical doc + fixtures proving parity for Node/Bun/Python (and any Java/.NET container behaviors where relevant). | +| 4 | SCAN-PROG-408-004 | DONE | Unblocks Bun sprint DOING. | Project Mgmt + Scanner Guild | **Create missing Bun analyzer charter**: add `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` synthesizing constraints from `docs/modules/scanner/architecture.md` and this sprint + `SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md`. Must include: allowed directories, test strategy, determinism rules, identity/evidence conventions, and "no absolute paths" requirement. | | 5 | SCAN-PROG-408-JAVA | TODO | Actions 1–2 recommended before emission format changes. | Java Analyzer Guild + QA Guild | **Implement all Java gaps** per `docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md`: (a) embedded libs inside fat archives without extraction, (b) `pom.xml` fallback when properties missing, (c) multi-module Gradle lock discovery + deterministic precedence, (d) runtime image component emission from `release`, (e) replace JNI string scanning with bytecode-based JNI analysis. Acceptance: Java analyzer tests + new fixtures/goldens; bounded scanning with explicit skipped markers. | | 6 | SCAN-PROG-408-DOTNET | TODO | Actions 1–2 recommended before adding declared-only identities. | .NET Analyzer Guild + QA Guild | **Implement all .NET gaps** per `docs/implplan/SPRINT_0404_0001_0001_scanner_dotnet_detection_gaps.md`: (a) declared-only fallback when no deps.json, (b) non-colliding identity for unresolved versions, (c) deterministic merge of declared vs installed packages, (d) bounded bundling signals, (e) optional declared edges provenance, (f) fixtures/docs (and optional bench). Acceptance: `.NET` analyzer emits components for source trees with lock/build files; no restore/MSBuild execution; deterministic outputs. | | 7 | SCAN-PROG-408-PYTHON | TODO | Actions 1–3 recommended before overlay/identity changes. | Python Analyzer Guild + QA Guild | **Implement all Python gaps** per `docs/implplan/SPRINT_0405_0001_0001_scanner_python_detection_gaps.md`: (a) layout-aware discovery (avoid “any dist-info anywhere”), (b) expanded lock/requirements parsing (includes/editables/PEP508/direct refs), (c) correct container overlay/whiteout semantics (or explicit overlayIncomplete markers), (d) vendored dependency surfacing with safe identity rules, (e) optional used-by signals (bounded/opt-in), (f) fixtures/docs/bench. Acceptance: deterministic fixtures for lock formats and container overlays; no invalid “editable-as-version” PURLs per Action 1. | @@ -72,10 +72,10 @@ ## Action Tracker | # | Action | Owner | Due (UTC) | Status | Notes | | --- | --- | --- | --- | --- | --- | -| 1 | Choose canonical doc path + define explicit-key identity recipe across analyzers. | Project Mgmt + Scanner Guild + Security Guild | 2025-12-13 | Open | Must prevent collisions with concrete PURLs; must be OS-path stable and deterministic. | -| 2 | Define evidence locator formats (lock entries, nested artifacts, derived evidence) and required hashing rules/bounds. | Project Mgmt + Scanner Guild + Export/UI/CLI Consumers | 2025-12-13 | Open | Must be parseable and stable; add golden fixtures asserting exact strings. | -| 3 | Define container layer/rootfs discovery + overlay semantics contract and bounds. | Project Mgmt + Scanner Guild | 2025-12-13 | Open | Align Node/Bun/Python; clarify when overlayIncomplete markers are required. | -| 4 | Create `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` and link it from Bun sprint prerequisites. | Project Mgmt | 2025-12-13 | Open | Required before Bun implementation tasks can flip to DOING. | +| 1 | Choose canonical doc path + define explicit-key identity recipe across analyzers. | Project Mgmt + Scanner Guild + Security Guild | 2025-12-13 | In Progress | Doc: `docs/modules/scanner/language-analyzers-contract.md`; Node/Bun/Python updated to emit explicit-key for non-concrete identities with tests/fixtures. | +| 2 | Define evidence locator formats (lock entries, nested artifacts, derived evidence) and required hashing rules/bounds. | Project Mgmt + Scanner Guild + Export/UI/CLI Consumers | 2025-12-13 | In Progress | Doc: `docs/modules/scanner/language-analyzers-contract.md`; Node/Bun/Python fixtures assert locator formats (lock entries, nested artifacts, derived evidence). | +| 3 | Define container layer/rootfs discovery + overlay semantics contract and bounds. | Project Mgmt + Scanner Guild | 2025-12-13 | In Progress | Doc: `docs/modules/scanner/language-analyzers-contract.md`; fixtures now cover Node/Bun/Python parity for `layers/`, `.layers/`, and `layer*/`. | +| 4 | Create `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md` and link it from Bun sprint prerequisites. | Project Mgmt | 2025-12-13 | Done | Created `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md`; updated Bun sprint prerequisites. | ## Decisions & Risks - **Decision (pending):** cross-analyzer identity/evidence/container contracts (Actions 1–3). @@ -92,4 +92,7 @@ | Date (UTC) | Update | Owner | | --- | --- | --- | | 2025-12-12 | Program sprint created to coordinate implementation of all language analyzer detection gaps (Java/.NET/Python/Node/Bun) with shared contracts and acceptance evidence. | Project Mgmt | +| 2025-12-13 | Created Bun analyzer charter (`src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md`); updated Bun sprint prerequisites; marked SCAN-PROG-408-004 complete. | Project Mgmt | +| 2025-12-13 | Set SCAN-PROG-408-001..003 to DOING; started Actions 1-3 (identity/evidence/container contracts). | Scanner Guild | +| 2025-12-13 | Implemented Node/Python contract compliance (explicit-key for declared-only, tarball/git/file/workspace classification; Python editable lock entries now explicit-key with host-path scrubbing) and extended fixtures for `.layers`/`layers`/`layer*`; Node + Python test suites passing. | Implementer | diff --git a/docs/implplan/SPRINT_0409_0001_0001_scanner_non_language_scanners_quality.md b/docs/implplan/SPRINT_0409_0001_0001_scanner_non_language_scanners_quality.md index efbc7ec6f..3338b8fd9 100644 --- a/docs/implplan/SPRINT_0409_0001_0001_scanner_non_language_scanners_quality.md +++ b/docs/implplan/SPRINT_0409_0001_0001_scanner_non_language_scanners_quality.md @@ -28,6 +28,7 @@ | 6 | SCAN-NL-0409-006 | DONE | — | Scanner · Backend | RPM sqlite read path: avoid `SELECT *` and column-scanning where feasible (schema probe + targeted column selection). Add unit coverage for schema variants. | | 7 | SCAN-NL-0409-007 | DONE | — | Scanner · Backend/QA | Native “unknowns” quality: emit unknowns even when dependency list is empty; extract ELF `.dynsym` undefined symbols for unknown edges; add regression test. | | 8 | SCAN-NL-0409-008 | DONE | — | Scanner · Docs | Document OS analyzer evidence semantics (paths/digests/warnings) and caching behavior under `docs/modules/scanner/` (and link from sprint Decisions & Risks). | +| 9 | SCAN-NL-0409-009 | DOING | Update Ruby analyzer determinism fixtures | Scanner · Backend/QA | Keep `src/Scanner/StellaOps.Scanner.sln` green: fix Ruby capability detection regression (`Open3.capture3`) and refresh Ruby golden fixtures (legacy/container/complex). | ## Execution Log | Date (UTC) | Update | Owner | @@ -39,6 +40,7 @@ | 2025-12-12 | Optimized rpmdb sqlite reader (schema probe + targeted selection/query); added tests. | Scanner | | 2025-12-12 | Improved native “unknowns” (ELF `.dynsym` undefined symbols) and added regression test. | Scanner | | 2025-12-12 | Documented OS/non-language evidence contract and caching behavior. | Scanner | +| 2025-12-13 | Follow-up QA: started SCAN-NL-0409-009 to keep Scanner solution tests green (Ruby analyzer determinism + capability regression). | Scanner | ## Decisions & Risks - **OS cache safety:** Only cache when the rootfs fingerprint is representative of analyzer inputs; otherwise bypass cache to avoid stale results. @@ -47,4 +49,5 @@ - **Evidence contract:** `docs/modules/scanner/os-analyzers-evidence.md`. ## Next Checkpoints -- 2025-12-12: Sprint completed; all tasks set to DONE. +- 2025-12-12: Sprint completed; all OS/non-language tasks set to DONE. +- 2025-12-13: Follow-up QA (SCAN-NL-0409-009) in progress. diff --git a/docs/implplan/SPRINT_0410_0001_0001_entrypoint_detection_reengineering_program.md b/docs/implplan/SPRINT_0410_0001_0001_entrypoint_detection_reengineering_program.md new file mode 100644 index 000000000..46b4ba8b0 --- /dev/null +++ b/docs/implplan/SPRINT_0410_0001_0001_entrypoint_detection_reengineering_program.md @@ -0,0 +1,159 @@ +# Sprint 0410.0001.0001 - Entrypoint Detection Re-Engineering Program + +## Topic & Scope +- Window: 2025-12-16 -> 2026-02-28 (UTC); phased delivery across 5 child sprints. +- **Vision:** Re-engineer entrypoint detection to be industry-leading with semantic understanding, temporal tracking, multi-container mesh analysis, speculative execution, binary intelligence, and predictive risk scoring. +- **Strategic Goal:** Position StellaOps entrypoint detection as the foundation for context-aware vulnerability assessment - answering not just "what's installed" but "what's running, how it's invoked, and what can reach it." +- **Working directory:** `docs/implplan` (coordination); implementation in `src/Scanner/__Libraries/StellaOps.Scanner.EntryTrace/` and related modules. + +## Program Architecture + +### Current State +The existing entrypoint detection has: +- Container-level OCI config parsing (ENTRYPOINT/CMD) +- ShellFlow static analyzer for shell scripts +- Per-language analyzers (Python, Java, Node, .NET, Go, Ruby, Rust, Bun, Deno, PHP) +- Evidence chains with `usedByEntrypoint` flags +- Dual-mode (static image + running container) + +### Target State: Entrypoint Knowledge Graph + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ ENTRYPOINT KNOWLEDGE GRAPH │ +├────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Semantic │────▶│ Temporal │────▶│ Mesh │ │ +│ │ Engine │ │ Graph │ │ Analysis │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Speculative │────▶│ Binary │────▶│ Predictive │ │ +│ │ Execution │ │ Intelligence │ │ Risk │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +│ │ +│ Query: "Which images have Django entrypoints reachable to │ +│ log4j 2.14.1?" │ +│ Answer: 847 images, 12 in production, 3 internet-facing │ +│ │ +└────────────────────────────────────────────────────────────────────┘ +``` + +## Child Sprints + +| Sprint ID | Name | Focus | Window | Status | +|-----------|------|-------|--------|--------| +| 0411.0001.0001 | Semantic Entrypoint Engine | Semantic understanding, intent/capability inference | 2025-12-16 -> 2025-12-30 | TODO | +| 0412.0001.0001 | Temporal & Mesh Entrypoint | Temporal tracking, multi-container mesh | 2026-01-02 -> 2026-01-17 | TODO | +| 0413.0001.0001 | Speculative Execution Engine | Symbolic execution, path enumeration | 2026-01-20 -> 2026-02-03 | TODO | +| 0414.0001.0001 | Binary Intelligence | Fingerprinting, symbol recovery | 2026-02-06 -> 2026-02-17 | TODO | +| 0415.0001.0001 | Predictive Risk Scoring | Risk-aware scoring, business context | 2026-02-20 -> 2026-02-28 | TODO | + +## Dependencies & Concurrency +- Upstream: Sprint 0401 Reachability Evidence Chain (completed tasks for richgraph-v1, symbol_id, code_id). +- Upstream: Sprint 0408 Scanner Language Detection Gaps Program (mature language analyzers). +- Child sprints 0411-0413 can proceed in parallel after semantic foundation lands. +- Sprints 0414-0415 depend on earlier sprints for data structures but can overlap. + +## Documentation Prerequisites +- docs/modules/scanner/architecture.md +- docs/modules/scanner/operations/entrypoint-problem.md +- docs/modules/scanner/operations/entrypoint-static-analysis.md +- docs/modules/scanner/operations/entrypoint-shell-analysis.md +- docs/modules/scanner/operations/entrypoint-runtime-overview.md +- docs/reachability/function-level-evidence.md +- docs/reachability/lattice.md +- src/Scanner/__Libraries/StellaOps.Scanner.EntryTrace/AGENTS.md (to be created) + +## Key Deliverables + +### Phase 1: Semantic Foundation (Sprint 0411) +1. **SemanticEntrypoint** record with intent, capabilities, attack surface +2. **ApplicationIntent** enumeration (web-server, cli-tool, batch-job, worker, serverless, etc.) +3. **CapabilityClass** enumeration (network-listen, file-write, exec-spawn, crypto, etc.) +4. **ThreatVector** inference from entrypoint characteristics +5. Cross-language semantic detection adapters + +### Phase 2: Temporal & Mesh (Sprint 0412) +1. **TemporalEntrypointGraph** for version-to-version tracking +2. **EntrypointDrift** detection and alerting +3. **MeshEntrypointGraph** for multi-container orchestration +4. **CrossContainerPath** reachability across services +5. Kubernetes/Compose manifest parsing + +### Phase 3: Speculative Execution (Sprint 0413) +1. **SymbolicExecutionEngine** for ShellFlow enhancement +2. **PathEnumerator** for all terminal states +3. **ConstraintSolver** for complex conditionals +4. **BranchCoverage** metrics and confidence + +### Phase 4: Binary Intelligence (Sprint 0414) +1. **CodeFingerprint** index from OSS package corpus +2. **SymbolRecovery** for stripped binaries +3. **SourceCorrelation** service +4. **FunctionSignatureInference** from binary analysis + +### Phase 5: Predictive Risk (Sprint 0415) +1. **RiskFactorExtractor** pipeline +2. **EntrypointRiskScorer** with business context +3. **AttackSurfaceQuantifier** per entrypoint +4. **EntrypointAsCode** auto-generated specifications + +## Competitive Differentiation + +| Capability | StellaOps (Target) | Competition | +|------------|-------------------|-------------| +| Semantic understanding | Full intent + capability inference | Pattern matching only | +| Temporal tracking | Version-to-version evolution | Snapshot only | +| Multi-container | Full mesh with cross-container reachability | Single container | +| Stripped binaries | Fingerprint + ML recovery | Limited/none | +| Speculative execution | All paths enumerated symbolically | Best-effort heuristics | +| Entrypoint-as-Code | Auto-generated, executable specs | Manual documentation | +| Predictive risk | Business-context-aware scoring | Static CVSS only | + +## Wave Coordination +| Wave | Child Sprints | Shared Prerequisites | Status | Notes | +|------|---------------|----------------------|--------|-------| +| Foundation | 0411 | Sprint 0401 richgraph/symbol contracts | TODO | Must land before other phases | +| Parallel | 0412, 0413 | 0411 semantic records | TODO | Can run concurrently | +| Intelligence | 0414 | 0411-0413 data structures | TODO | Binary focus | +| Risk | 0415 | 0411-0414 evidence chains | TODO | Final phase | + +## Interlocks +- Semantic record schema (Sprint 0411) must stabilize before Temporal/Mesh (0412) or Speculative (0413) start. +- Binary fingerprint corpus (Sprint 0414) requires OSS package index integration. +- Risk scoring (Sprint 0415) needs Policy Engine integration for gate enforcement. +- All phases emit to richgraph-v1 with BLAKE3 hashing per CONTRACT-RICHGRAPH-V1-015. + +## Upcoming Checkpoints +- 2025-12-16 - Sprint 0411 kickoff; semantic schema draft review. +- 2025-12-23 - Sprint 0411 midpoint; ApplicationIntent/CapabilityClass enums frozen. +- 2025-12-30 - Sprint 0411 close; semantic foundation ready for 0412/0413. +- 2026-01-02 - Sprints 0412/0413 kickoff (parallel). +- 2026-02-28 - Program close; all phases delivered. + +## Action Tracker +| # | Action | Owner | Due (UTC) | Status | Notes | +|---|--------|-------|-----------|--------|-------| +| 1 | Create AGENTS.md for EntryTrace module | Scanner Guild | 2025-12-16 | TODO | Foundation for implementers | +| 2 | Draft SemanticEntrypoint schema | Scanner Guild | 2025-12-18 | TODO | Phase 1 core deliverable | +| 3 | Define ApplicationIntent enumeration | Scanner Guild | 2025-12-20 | TODO | Needs cross-language input | +| 4 | Create temporal graph storage design | Platform Guild | 2026-01-02 | TODO | Phase 2 dependency | +| 5 | Evaluate binary fingerprint corpus options | Scanner Guild | 2026-02-01 | TODO | Phase 4 dependency | + +## Decisions & Risks + +| ID | Risk | Impact | Mitigation / Owner | +|----|------|--------|-------------------| +| R1 | Semantic schema changes mid-program | Rework in dependent phases | Freeze schema by Sprint 0411 close; Scanner Guild | +| R2 | Binary fingerprint corpus size/latency | Slow startup, large storage | Use lazy loading, tiered caching; Platform Guild | +| R3 | Multi-container mesh complexity | Detection gaps in complex K8s | Phased support; start with common patterns; Scanner Guild | +| R4 | Speculative execution path explosion | Performance issues | Add depth limits, caching; Scanner Guild | +| R5 | Risk scoring model accuracy | False confidence signals | Train on CVE exploitation data; validate with red team; Signals Guild | + +## Execution Log +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2025-12-13 | Created program sprint from strategic analysis; outlined 5 child sprints with phased delivery; defined competitive differentiation matrix. | Planning | diff --git a/docs/implplan/SPRINT_0411_0001_0001_semantic_entrypoint_engine.md b/docs/implplan/SPRINT_0411_0001_0001_semantic_entrypoint_engine.md new file mode 100644 index 000000000..e2ecc4455 --- /dev/null +++ b/docs/implplan/SPRINT_0411_0001_0001_semantic_entrypoint_engine.md @@ -0,0 +1,163 @@ +# Sprint 0411.0001.0001 - Semantic Entrypoint Engine + +## Topic & Scope +- Window: 2025-12-16 -> 2025-12-30 (UTC); foundation phase for entrypoint re-engineering. +- Build semantic understanding layer that infers intent, capabilities, and attack surface from entrypoints. +- Enable downstream phases (temporal, mesh, speculative, binary, risk) with stable data structures. +- **Working directory:** `src/Scanner/__Libraries/StellaOps.Scanner.EntryTrace/Semantic/` + +## Dependencies & Concurrency +- Upstream: Sprint 0401 Reachability Evidence Chain (richgraph-v1, symbol_id, code_id contracts). +- Upstream: Sprint 0408 Language Detection Gaps (mature Python/Java/Node analyzers). +- Blocks: Sprints 0412-0415 depend on semantic records from this sprint. +- Language-specific adapters can be developed in parallel once core schema lands. + +## Documentation Prerequisites +- docs/modules/scanner/operations/entrypoint-problem.md +- docs/modules/scanner/operations/entrypoint-static-analysis.md +- docs/modules/scanner/operations/entrypoint-lang-*.md (per-language guides) +- docs/reachability/function-level-evidence.md +- src/Scanner/__Libraries/StellaOps.Scanner.EntryTrace/AGENTS.md + +## Delivery Tracker +| # | Task ID | Status | Key dependency / next step | Owners | Task Definition | +|---|---------|--------|---------------------------|--------|-----------------| +| 1 | ENTRY-SEM-411-001 | TODO | None; foundation task | Scanner Guild | Create `SemanticEntrypoint` record with Id, Specification, Intent, Capabilities, AttackSurface, DataBoundaries, Confidence fields. | +| 2 | ENTRY-SEM-411-002 | TODO | Task 1 | Scanner Guild | Define `ApplicationIntent` enumeration: WebServer, CliTool, BatchJob, Worker, Serverless, Daemon, InitSystem, Supervisor, DatabaseServer, MessageBroker, CacheServer, ProxyGateway, Unknown. | +| 3 | ENTRY-SEM-411-003 | TODO | Task 1 | Scanner Guild | Define `CapabilityClass` enumeration: NetworkListen, NetworkConnect, FileRead, FileWrite, ProcessSpawn, CryptoOperation, DatabaseAccess, MessageQueue, CacheAccess, ExternalApi, UserInput, ConfigLoad, SecretAccess, LogEmit. | +| 4 | ENTRY-SEM-411-004 | TODO | Task 1 | Scanner Guild | Define `ThreatVector` record with VectorType (Ssrf, Sqli, Xss, Rce, PathTraversal, Deserialization, TemplateInjection, AuthBypass, InfoDisclosure, Dos), Confidence, Evidence, EntryPath. | +| 5 | ENTRY-SEM-411-005 | TODO | Task 1 | Scanner Guild | Define `DataFlowBoundary` record with BoundaryType (HttpRequest, HttpResponse, FileInput, FileOutput, DatabaseQuery, MessageReceive, MessageSend, EnvironmentVar, CommandLineArg), Direction, Sensitivity. | +| 6 | ENTRY-SEM-411-006 | TODO | Task 1 | Scanner Guild | Define `SemanticConfidence` record with Score (0.0-1.0), Tier (Definitive, High, Medium, Low, Unknown), ReasoningChain (list of evidence strings). | +| 7 | ENTRY-SEM-411-007 | TODO | Tasks 1-6 | Scanner Guild | Create `ISemanticEntrypointAnalyzer` interface with `AnalyzeAsync(EntryTraceResult, LanguageAnalyzerResult, CancellationToken) -> SemanticEntrypoint`. | +| 8 | ENTRY-SEM-411-008 | TODO | Task 7 | Scanner Guild | Implement `PythonSemanticAdapter` inferring intent from: Django (WebServer), Celery (Worker), Click/Typer (CliTool), Lambda (Serverless), Flask/FastAPI (WebServer). | +| 9 | ENTRY-SEM-411-009 | TODO | Task 7 | Scanner Guild | Implement `JavaSemanticAdapter` inferring intent from: Spring Boot (WebServer), Quarkus (WebServer), Micronaut (WebServer), Kafka Streams (Worker), Main-Class patterns. | +| 10 | ENTRY-SEM-411-010 | TODO | Task 7 | Scanner Guild | Implement `NodeSemanticAdapter` inferring intent from: Express/Koa/Fastify (WebServer), CLI bin entries (CliTool), worker threads, Lambda handlers (Serverless). | +| 11 | ENTRY-SEM-411-011 | TODO | Task 7 | Scanner Guild | Implement `DotNetSemanticAdapter` inferring intent from: ASP.NET Core (WebServer), Console apps (CliTool), Worker services (Worker), Azure Functions (Serverless). | +| 12 | ENTRY-SEM-411-012 | TODO | Task 7 | Scanner Guild | Implement `GoSemanticAdapter` inferring intent from: net/http patterns (WebServer), cobra/urfave CLI (CliTool), gRPC servers, main package analysis. | +| 13 | ENTRY-SEM-411-013 | TODO | Tasks 8-12 | Scanner Guild | Create `CapabilityDetector` that analyzes imports/dependencies to infer capabilities (e.g., `import socket` -> NetworkConnect, `import os.path` -> FileRead). | +| 14 | ENTRY-SEM-411-014 | TODO | Task 13 | Scanner Guild | Create `ThreatVectorInferrer` that maps capabilities and framework patterns to likely attack vectors (e.g., WebServer + DatabaseAccess + UserInput -> Sqli risk). | +| 15 | ENTRY-SEM-411-015 | TODO | Task 13 | Scanner Guild | Create `DataBoundaryMapper` that traces data flow edges from entrypoint through framework handlers to I/O boundaries. | +| 16 | ENTRY-SEM-411-016 | TODO | Tasks 7-15 | Scanner Guild | Create `SemanticEntrypointOrchestrator` that composes adapters, detectors, and inferrers into unified semantic analysis pipeline. | +| 17 | ENTRY-SEM-411-017 | TODO | Task 16 | Scanner Guild | Integrate semantic analysis into `EntryTraceAnalyzer` post-processing, emit `SemanticEntrypoint` alongside `EntryTraceResult`. | +| 18 | ENTRY-SEM-411-018 | TODO | Task 17 | Scanner Guild | Add semantic fields to `LanguageComponentRecord`: `intent`, `capabilities[]`, `threatVectors[]`. | +| 19 | ENTRY-SEM-411-019 | TODO | Task 18 | Scanner Guild | Update richgraph-v1 schema to include semantic metadata on entrypoint nodes. | +| 20 | ENTRY-SEM-411-020 | TODO | Task 19 | Scanner Guild | Add CycloneDX and SPDX property extensions for semantic entrypoint data. | +| 21 | ENTRY-SEM-411-021 | TODO | Tasks 8-12 | QA Guild | Create test fixtures for each language semantic adapter with expected intent/capabilities. | +| 22 | ENTRY-SEM-411-022 | TODO | Task 21 | QA Guild | Add golden test suite validating semantic analysis determinism. | +| 23 | ENTRY-SEM-411-023 | TODO | Task 22 | Docs Guild | Document semantic entrypoint schema in `docs/modules/scanner/operations/entrypoint-semantic.md`. | +| 24 | ENTRY-SEM-411-024 | TODO | Task 23 | Docs Guild | Update `docs/modules/scanner/architecture.md` with semantic analysis pipeline. | +| 25 | ENTRY-SEM-411-025 | TODO | Task 24 | CLI Guild | Add `stella scan --semantic` flag and semantic output fields to JSON/table formats. | + +## Wave Coordination +| Wave | Tasks | Shared Prerequisites | Status | Notes | +|------|-------|---------------------|--------|-------| +| Schema Definition | 1-6 | None | TODO | Core data structures | +| Adapter Interface | 7 | Schema frozen | TODO | Contract for language adapters | +| Language Adapters | 8-12 | Interface defined | TODO | Can run in parallel | +| Cross-Cutting Analysis | 13-15 | Adapters started | TODO | Capability/threat/boundary detection | +| Integration | 16-20 | Adapters + analysis | TODO | Wire into scanner pipeline | +| QA & Docs | 21-25 | Integration complete | TODO | Validation and documentation | + +## Interlocks +- Schema tasks (1-6) must complete before interface task (7). +- Interface task (7) gates all language adapters (8-12). +- Language adapters can proceed in parallel. +- Cross-cutting analysis (13-15) can start once any adapter is in progress. +- Integration tasks (16-20) require most adapters complete. +- QA/Docs (21-25) can overlap with late integration. + +## Upcoming Checkpoints +- 2025-12-18 - Schema freeze (tasks 1-6 complete); interface draft (task 7). +- 2025-12-23 - Language adapters midpoint (tasks 8-12 in progress); cross-cutting analysis started. +- 2025-12-27 - Integration tasks started (tasks 16-20). +- 2025-12-30 - Sprint close; semantic foundation ready. + +## Action Tracker +| # | Action | Owner | Due (UTC) | Status | Notes | +|---|--------|-------|-----------|--------|-------| +| 1 | Review existing entrypoint detection code | Scanner Guild | 2025-12-16 | TODO | Understand integration points | +| 2 | Draft ApplicationIntent enum with cross-team input | Scanner Guild | 2025-12-17 | TODO | Need input from all language teams | +| 3 | Create AGENTS.md for EntryTrace module | Scanner Guild | 2025-12-16 | TODO | Implementer guidance | +| 4 | Validate semantic schema against richgraph-v1 | Platform Guild | 2025-12-18 | TODO | Ensure compatibility | + +## Decisions & Risks + +| ID | Risk | Impact | Mitigation / Owner | +|----|------|--------|-------------------| +| R1 | Intent enumeration incomplete | Missing application types | Start with common patterns; extend as needed; Scanner Guild | +| R2 | Capability detection false positives | Noise in attack surface | Use confidence scoring; require multiple signals; Scanner Guild | +| R3 | Schema changes after freeze | Rework in dependent sprints | Strict freeze enforcement after 2025-12-18; Planning | +| R4 | Language adapter coverage gaps | Inconsistent semantic depth | Prioritize Python/Java/Node; others can be stubs; Scanner Guild | + +## Schema Preview + +### SemanticEntrypoint Record +```csharp +public sealed record SemanticEntrypoint +{ + public required string Id { get; init; } + public required EntrypointSpecification Specification { get; init; } + public required ApplicationIntent Intent { get; init; } + public required ImmutableArray Capabilities { get; init; } + public required ImmutableArray AttackSurface { get; init; } + public required ImmutableArray DataBoundaries { get; init; } + public required SemanticConfidence Confidence { get; init; } + public ImmutableDictionary? Metadata { get; init; } +} +``` + +### ApplicationIntent Enumeration +```csharp +public enum ApplicationIntent +{ + Unknown = 0, + WebServer = 1, // HTTP/HTTPS listener (Django, Express, ASP.NET) + CliTool = 2, // Command-line utility (Click, Cobra) + BatchJob = 3, // One-shot data processing + Worker = 4, // Background job processor (Celery, Sidekiq) + Serverless = 5, // FaaS handler (Lambda, Azure Functions) + Daemon = 6, // Long-running background service + InitSystem = 7, // Process manager (systemd, s6) + Supervisor = 8, // Child process supervisor + DatabaseServer = 9, // Database engine + MessageBroker = 10, // Message queue server + CacheServer = 11, // Cache/session store + ProxyGateway = 12, // Reverse proxy, API gateway + TestRunner = 13, // Test framework execution + DevServer = 14, // Development-only server +} +``` + +### CapabilityClass Enumeration +```csharp +[Flags] +public enum CapabilityClass : long +{ + None = 0, + NetworkListen = 1 << 0, // Opens listening socket + NetworkConnect = 1 << 1, // Makes outbound connections + FileRead = 1 << 2, // Reads from filesystem + FileWrite = 1 << 3, // Writes to filesystem + ProcessSpawn = 1 << 4, // Spawns child processes + CryptoOperation = 1 << 5, // Encryption/signing operations + DatabaseAccess = 1 << 6, // Database client operations + MessageQueue = 1 << 7, // Message broker client + CacheAccess = 1 << 8, // Cache client operations + ExternalApi = 1 << 9, // External HTTP API calls + UserInput = 1 << 10, // Accepts user input + ConfigLoad = 1 << 11, // Loads configuration files + SecretAccess = 1 << 12, // Accesses secrets/credentials + LogEmit = 1 << 13, // Emits logs + MetricsEmit = 1 << 14, // Emits metrics/telemetry + SystemCall = 1 << 15, // Makes privileged syscalls + ContainerEscape = 1 << 16, // Capabilities enabling escape + KernelModule = 1 << 17, // Loads kernel modules + Ptrace = 1 << 18, // Process tracing + RawSocket = 1 << 19, // Raw network access +} +``` + +## Execution Log +| Date (UTC) | Update | Owner | +|------------|--------|-------| +| 2025-12-13 | Created sprint from program sprint 0410; defined 25 tasks across schema, adapters, integration, QA/docs; included schema previews. | Planning | diff --git a/docs/implplan/SPRINT_0215_0001_0001_vuln_triage_ux.md b/docs/implplan/archived/SPRINT_0215_0001_0001_vuln_triage_ux.md similarity index 97% rename from docs/implplan/SPRINT_0215_0001_0001_vuln_triage_ux.md rename to docs/implplan/archived/SPRINT_0215_0001_0001_vuln_triage_ux.md index e657739e0..44065c0cc 100644 --- a/docs/implplan/SPRINT_0215_0001_0001_vuln_triage_ux.md +++ b/docs/implplan/archived/SPRINT_0215_0001_0001_vuln_triage_ux.md @@ -110,9 +110,9 @@ | 1 | Finalize VEX decision schema with Excititor team | Platform Guild | 2025-12-02 | DONE | | 2 | Confirm attestation predicate types with Attestor team | API Guild | 2025-12-03 | DONE | | 3 | Review audit bundle format with Export Center team | API Guild | 2025-12-04 | DONE | -| 4 | Accessibility review of VEX modal with Accessibility Guild | UI Guild | 2025-12-09 | TODO | +| 4 | Accessibility review of VEX modal with Accessibility Guild | UI Guild | 2025-12-09 | DONE | | 5 | Align UI work to canonical workspace `src/Web/StellaOps.Web` | DevEx · UI Guild | 2025-12-06 | DONE | -| 6 | Regenerate deterministic fixtures for triage/VEX components (tests/e2e/offline-kit) | DevEx · UI Guild | 2025-12-13 | TODO | +| 6 | Regenerate deterministic fixtures for triage/VEX components (tests/e2e/offline-kit) | DevEx · UI Guild | 2025-12-13 | DONE | ## Decisions & Risks | Risk | Impact | Mitigation / Next Step | @@ -138,6 +138,7 @@ | 2025-12-12 | Normalized prerequisites to archived advisory/sprint paths; aligned API endpoint paths and Wave A deliverables to `src/Web/StellaOps.Web`. | Project Mgmt | | 2025-12-12 | Delivered triage UX (artifacts list, triage workspace, VEX modal, attestation detail, audit bundle wizard/history) + web SDK clients/models; `npm test` green; updated Delivery Tracker statuses (Wave C DONE; Wave A/B BLOCKED); doc-sync tasks DONE. | Implementer | | 2025-12-12 | Synced sprint tracker to implementation: Wave A/B (SCHEMA-08-*, DTO-09-*, API-VEX-06-*, API-AUDIT-07-*) and TRIAGE-GAPS-215-042 / UI-PROOF-VEX-0215-010 / TTE-GAPS-0215-011 now DONE; Action Tracker #1-3 DONE; remaining Action Tracker #4 and #6. | Implementer | +| 2025-12-13 | Completed Action Tracker #4/#6: Playwright Axe a11y smoke passes in strict mode for triage VEX modal (`src/Web/StellaOps.Web/tests/e2e/a11y-smoke.spec.ts`) and graph severity filter label is now associated (`src/Web/StellaOps.Web/src/app/features/graph/graph-explorer.component.html`); triage quickstart fixtures remain deterministic via mock clients (`src/Web/StellaOps.Web/src/app/core/api/vex-decisions.client.ts`, `src/Web/StellaOps.Web/src/app/core/api/audit-bundles.client.ts`). | Implementer | --- *Sprint created: 2025-11-28* diff --git a/docs/modules/scanner/README.md b/docs/modules/scanner/README.md index 445751b90..21fea8062 100644 --- a/docs/modules/scanner/README.md +++ b/docs/modules/scanner/README.md @@ -4,7 +4,7 @@ Scanner analyses container images layer-by-layer, producing deterministic SBOM f ## Latest updates (2025-12-12) - Deterministic SBOM composition fixture published at `docs/modules/scanner/fixtures/deterministic-compose/` with DSSE, `_composition.json`, BOM, and hashes; doc `deterministic-sbom-compose.md` promoted to Ready v1.0 with offline verification steps. -- Node analyzer now ingests npm/yarn/pnpm lockfiles, emitting `DeclaredOnly` components with lock provenance. The CLI companion command `stella node lock-validate` runs the collector offline, surfaces declared-only or missing-lock packages, and emits telemetry via `stellaops.cli.node.lock_validate.count`. +- Node analyzer now ingests npm/yarn/pnpm lockfiles, emitting `DeclaredOnly` components with lock provenance. The CLI companion command `stella node lock-validate` runs the collector offline, surfaces declared-only or missing-lock packages, and emits telemetry via `stellaops.cli.node.lock_validate.count`. See `docs/modules/scanner/analyzers-node.md` and bench scenario `node_detection_gaps_fixture`. - Python analyzer picks up `requirements*.txt`, `Pipfile.lock`, and `poetry.lock`, tagging installed distributions with lock provenance and generating declared-only components for policy. Use `stella python lock-validate` to run the same checks locally before images are built. - Java analyzer now parses `gradle.lockfile`, `gradle/dependency-locks/**/*.lockfile`, and `pom.xml` dependencies via the new `JavaLockFileCollector`, merging lock metadata onto jar evidence and emitting declared-only components when jars are absent. The new CLI verb `stella java lock-validate` reuses that collector offline (table/JSON output) and records `stellaops.cli.java.lock_validate.count{outcome}` for observability. - Worker/WebService now resolve cache roots and feature flags via `StellaOps.Scanner.Surface.Env`; misconfiguration warnings are documented in `docs/modules/scanner/design/surface-env.md` and surfaced through startup validation. @@ -37,6 +37,7 @@ Scanner analyses container images layer-by-layer, producing deterministic SBOM f - ./operations/analyzers-grafana-dashboard.json - ./operations/rustfs-migration.md - ./operations/entrypoint.md +- ./analyzers-node.md - ./operations/secret-leak-detection.md - ./operations/dsse-rekor-operator-guide.md - ./os-analyzers-evidence.md diff --git a/docs/modules/scanner/analyzers-bun.md b/docs/modules/scanner/analyzers-bun.md new file mode 100644 index 000000000..ff76d2610 --- /dev/null +++ b/docs/modules/scanner/analyzers-bun.md @@ -0,0 +1,81 @@ +# Bun Analyzer (Scanner) + +## What it does +- Inventories npm-ecosystem dependencies from Bun-managed projects without executing `bun`. +- Supports installed inventory (`node_modules/**/package.json`), lockfile-only inventory (`bun.lock`), and declared-only fallback from `package.json`. +- Enriches output with deterministic scope signals (`dev`, `optional`, `peer`, `scopeUnknown`), patch attribution, and bounded sha256 evidence. + +## Inputs and precedence +1. **Installed inventory** (`node_modules/` present): traverse installed packages and emit components from installed `package.json` (uses `bun.lock` for resolved/integrity + scope enrichment when present). +2. **Lockfile-only** (`bun.lock` present, no install): parse `bun.lock` and emit components from lock entries. +3. **Declared-only fallback** (project markers present but no `bun.lock`/install): emit explicit-key components from `package.json` dependency sections. +4. **Unsupported** (`bun.lockb` only): emit a remediation record explaining how to produce `bun.lock`. + +## Project discovery (including container roots) +The analyzer discovers Bun project roots under: +- The analysis root (`context.RootPath`) +- Common OCI unpack layouts: `layers/*`, `.layers/*`, and `layer*` (direct children) + +Discovery is bounded and deterministic: +- Sorted directory enumeration +- Explicit depth and root caps +- Never recurses into `node_modules/` + +## Identity rules (PURL vs explicit key) +Concrete versions emit a PURL: +- `purl = pkg:npm/@` +- Concrete versions follow the Node-style guardrail (no ranges/tags/paths embedded as a "version"; see `Internal/BunVersionSpec.IsConcreteNpmVersion`). + +Non-concrete versions emit an explicit key: +- `componentKey = explicit::::npm::::sha256:` +- `purl = null`, `version = null` +- Used for declared-only dependencies and any lock/installed records whose `version` is not concrete (e.g., `workspace:*`, `link:../...`, `file:../...`). + +Explicit-key digest input (canonical, UTF-8): +``` +npm\n\n\n +``` +Generated via `LanguageExplicitKey.Create(...)` and aligned with `docs/modules/scanner/language-analyzers-contract.md`. + +## Evidence and locators +All evidence locators are relative and use `/` separators. + +### File evidence +- Installed packages: `node_modules/.../package.json` +- Hashing: sha256 is computed for `package.json` only when size is within 1 MiB; when skipped, metadata includes: + - `packageJson.hashSkipped=true` + - `packageJson.hashSkipReason=...` + +### Lockfile entry evidence +- Locator format: `:packages[@]` + - Example: `bun.lock:packages[lodash@4.17.21]` +- Hashing: sha256 is computed for `bun.lock` only when size is within 50 MiB; when skipped, metadata includes: + - `bunLock.hashSkipped=true` + - `bunLock.hashSkipReason=...` + +## Scope semantics (dev/optional/peer) +Scope is derived deterministically from the `bun.lock` dependency graph rooted at `package.json` declarations: +- `dev=true` only when dev reachability is provable. +- `optional=true` and `peer=true` are preserved when present in lock data or derived from declared scopes. +- If the graph cannot disambiguate (multiple candidates/specifier mismatch), the record is marked: + - `scopeUnknown=true` + - `dev=false` (do not guess) + +`includeDev=false` filters only packages proven to be dev-only; unknown-scope packages are kept but marked `scopeUnknown=true`. + +## Patches and workspaces +- Workspace patterns come from root `package.json` (`workspaces`). +- Patch attribution supports Bun's `patchedDependencies` and patch directories. +- Patch keys preserve version specificity (`name@version`) and patch paths are emitted as deterministic project-relative paths. +- Patch matching precedence: `name@version` first; then name-only only when unambiguous. + +## Known limitations +- `bun.lockb` (binary lockfile) is not parsed; a remediation record is emitted instead. +- The analyzer does not execute `bun` and does not fetch registries; offline-only behavior is enforced. + +## References +- Sprint: `docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md` +- Cross-analyzer contract: `docs/modules/scanner/language-analyzers-contract.md` +- Design notes: `docs/modules/scanner/prep/bun-analyzer-design.md` +- Gotchas: `docs/modules/scanner/bun-analyzer-gotchas.md` + diff --git a/docs/modules/scanner/analyzers-java.md b/docs/modules/scanner/analyzers-java.md new file mode 100644 index 000000000..9c6c51576 --- /dev/null +++ b/docs/modules/scanner/analyzers-java.md @@ -0,0 +1,65 @@ +# Java Analyzer (Scanner) + +## What it does +- Inventories Maven coordinates from JVM archives (JAR/WAR/EAR/fat JAR) without executing build tools. +- Prefers installed artifact metadata (`META-INF/maven/**/pom.properties`), with a `pom.xml` fallback when properties are missing. +- Enriches output with bounded embedded-library scan metadata and JNI usage hints. + +## Inputs and precedence +1. **Installed archive inventory**: parse Maven coordinates from `META-INF/maven/**/pom.properties` in each discovered archive. +2. **`pom.xml` fallback**: when no `pom.properties` in the archive, parse `META-INF/maven/**/pom.xml` and emit a Maven PURL only when `groupId`, `artifactId`, and `version` are concrete (no placeholders like `${...}`). +3. **Lock augmentation (current)**: when a lock entry matches an installed artifact, merge lock metadata onto the component; unmatched lock entries still emit declared-only components. +4. **Multi-module lock precedence (pending)**: deterministic precedence rules are tracked in `SCAN-JAVA-403-003` (blocked). +5. **Runtime images (pending)**: runtime component identity is tracked in `SCAN-JAVA-403-004` (blocked). + +## Embedded archives (fat JAR / WAR / EAR layouts) +The analyzer scans embedded library jars without extracting them to disk: +- `BOOT-INF/lib/*.jar` +- `WEB-INF/lib/*.jar` +- `APP-INF/lib/*.jar` +- `lib/*.jar` + +### Locator format +Evidence locators are nested deterministically using `!` separators: +- `outer.jar!BOOT-INF/lib/inner.jar!META-INF/maven/.../pom.properties` + +### Bounds and skip markers +Embedded scanning is bounded and deterministic: +- Max embedded jars per archive: `256` +- Max embedded jar bytes: `25 MiB` + +When embedded scanning is skipped or truncated, the outer component metadata includes deterministic markers: +- `embeddedScan.candidateJars`, `embeddedScan.scannedJars`, `embeddedScan.emittedComponents` +- `embeddedScanSkipped=true`, `embeddedScan.skippedJars`, `embeddedScanSkipReasons=<...>` (when applicable) + +Embedded components include: +- `embedded=true` +- `embedded.containerJarPath=` +- `embedded.entryPath=` + +## Evidence and hashing +- Evidence locators are project-relative, use `/` separators, and use `!` for nested artifact paths. +- `sha256` for `pom.properties` and `pom.xml` evidence is computed over the raw entry bytes. + +## `pom.xml` with incomplete coordinates +When `pom.xml` is present but coordinates are incomplete (missing values or `${...}` placeholders), the analyzer emits an explicit-key component: +- `purl=null`, `version=null` +- `metadata.unresolvedCoordinates=true` +- `componentKey` follows the cross-analyzer explicit-key scheme via `LanguageExplicitKey.Create("java", "maven", ...)` + +## JNI metadata (bytecode-based) +JNI hints are derived from parsed bytecode (native method flags and load call sites), not raw ASCII scanning. + +When bytecode analysis finds JNI edges (`jni.edgeCount > 0`), components are annotated with bounded, deterministic metadata: +- `jni.edgeCount`, `jni.nativeMethodCount`, `jni.loadCallCount`, optional `jni.warningCount` +- `jni.reasons` (distinct reason codes) +- `jni.targetLibraries` (top-N stable sample; currently 12) + +## Known limitations +- Shaded jars that strip Maven metadata remain best-effort; embedded libs without Maven metadata do not emit components. +- Gradle multi-module lock precedence and runtime image component identity remain blocked until explicit decisions land. + +## References +- Sprint: `docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md` +- Cross-analyzer contract: `docs/modules/scanner/language-analyzers-contract.md` +- Implementation: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs` diff --git a/docs/modules/scanner/analyzers-node.md b/docs/modules/scanner/analyzers-node.md new file mode 100644 index 000000000..20ec38ca2 --- /dev/null +++ b/docs/modules/scanner/analyzers-node.md @@ -0,0 +1,79 @@ +# Node Analyzer (npm/Yarn/pnpm) + +This document captures the Node language analyzer’s deterministic behavior guarantees and safety constraints (what it emits, what it refuses to emit, and how it stays bounded/offline). + +## Component identity & precedence + +### Installed vs declared-only +- The analyzer always emits **on-disk inventory** first (workspace member manifests + installed `node_modules`/PNPM/Yarn PnP cache packages). +- It then emits **declared-only** components for lockfile / manifest declarations that are **not backed by on-disk inventory**: + - If a declared entry has a **concrete resolved version** from a lockfile, it emits a versioned `pkg:npm/...@` PURL. + - If the version is **non-concrete** (ranges/tags/git/file/workspace/link/path), it emits an **explicit-key** component (`purl=null`, `version=null`). + +### Identity safety (PURL vs explicit-key) +- Concrete PURLs are emitted only when the analyzer can prove a **concrete version** from local evidence (installed `package.json` or a lockfile-resolved entry). +- Declared-only/non-concrete dependencies use `LanguageExplicitKey` (see `docs/modules/scanner/language-analyzers-contract.md`). + +### Lock metadata lookup precedence +When attaching lock metadata to an installed package: +1) `package-lock.json` path match (`packages[""]`), +2) `(name, version)` match (Yarn/pnpm multi-version support), +3) fallback to name-only (last-wins) for legacy locks. + +## Lockfile parsing guarantees (offline) + +### `package-lock.json` (npm) +- Supports v3+ `packages{}` layout and legacy `dependencies{}` traversal. +- Correctly extracts nested names from `node_modules/.../node_modules/...` paths (including scoped packages). + +### `yarn.lock` (Yarn v1 + Berry v2/v3) +- Supports both Yarn v1 (`resolved "https://..."`) and Berry fields (`resolution:`, `checksum:`). +- If `integrity` is absent but `checksum` is present, the analyzer records integrity-like evidence as `checksum:`. +- Ignores the `__metadata` section. + +### `pnpm-lock.yaml` (pnpm) +- Parses modern `packages:` and `snapshots:` sections. +- Does not drop entries that lack `integrity` (workspace/link/file/git); instead it emits: + - `lockIntegrityMissing=true` + - `lockIntegrityMissingReason=` + +## Workspaces +- Reads workspace members from the root `package.json` (`workspaces` array or `{ packages: [...] }` form). +- Supports glob patterns: + - `*` (single segment) + - `**` (multi-segment) +- Expansion is bounded and deterministic: + - Skips `node_modules` + - Caps traversal depth and total visited directories/members + - Stable, sorted member output +- Dependency scopes (`production|development|peer|optional`) are derived from both the root and workspace manifests, with deterministic precedence. + +## Import scanning (bounded) +- Import scanning runs only for the root package and workspace member packages (not `node_modules` packages). +- File types: `.js/.jsx/.mjs/.cjs/.ts/.tsx/.mts/.cts`. +- Parser behavior: + - Attempts AST parsing as script/module; falls back to a bounded regex heuristic for TS when parsing fails. +- Hard caps per package: + - `maxFiles=500`, `maxBytes=5MiB`, `maxFileBytes=512KiB`, `maxDepth=20` + - Skips `node_modules` and `.pnpm` directories during traversal +- If capped, the analyzer marks the package metadata with: + - `importScanSkipped=true` + - `importScan.filesScanned=` + - `importScan.bytesScanned=` + +## Container layer layouts +- Candidate layer roots under the analysis root: + - `layers/*`, `.layers/*`, `layer*` +- Each candidate root is scanned independently. +- The analyzer also discovers `package.json` roots nested under layer roots (bounded depth) and includes their nested `node_modules` roots when present. + +## Determinism & evidence hashing +- On-disk `package.json` manifests are hashed (sha256) when ≤ 1 MiB and attached to the root evidence for deterministic provenance. +- Output ordering is stable (componentKey ordering, sorted metadata/evidence). + +## Benchmark +- Scenario id: `node_detection_gaps_fixture` (config: `src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json`) +- Fixture root: `samples/runtime/node-detection-gaps` +- Run: + - `dotnet run --project src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj -- --repo-root . --config src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json --json out/bench/scanner-analyzers/latest.json --prom out/bench/scanner-analyzers/latest.prom` + - Prometheus output includes additional metrics under `scanner_analyzer_bench_metric{scenario=\"...\",name=\"node.importScan.*\"}`. diff --git a/docs/modules/scanner/analyzers-python.md b/docs/modules/scanner/analyzers-python.md new file mode 100644 index 000000000..25b04cff4 --- /dev/null +++ b/docs/modules/scanner/analyzers-python.md @@ -0,0 +1,69 @@ +# Python Analyzer (Scanner) + +## What it does +- Inventories Python distributions without executing `python`/`pip` (static inspection only). +- Prefers installed distribution metadata (`*.dist-info/`) and validates `RECORD` when present (bounded, streaming IO). +- Emits deterministic component metadata (`pkg.kind`, `pkg.confidence`, `pkg.location`) and evidence locators for replay/audit. + +## Inputs and precedence +1. **Installed inventory (preferred)**: detect site-packages roots and parse `*.dist-info/` / `*.egg-info/` metadata for concrete `pkg:pypi/@` components. +2. **Archive inventory**: mount wheels (`*.whl`) and zipapps (`*.pyz`, `*.pyzw`) into the Python VFS and enrich any in-archive `*.dist-info/` metadata (including `RECORD` verification). +3. **Lock augmentation (current)**: parse root-level `requirements*.txt` pinned entries (`==`/`===`), `Pipfile.lock` `default` section, and `poetry.lock`; when a lock entry matches an installed component, merge lock metadata. +4. **Declared-only (current)**: lock entries not present in installed inventory still emit components: + - concrete versions emit a versioned `pkg:pypi/...@` PURL + - non-concrete declarations (e.g., editable paths) emit explicit-key components (see Identity Rules) + +## Project discovery (including container roots) +The analyzer is layout-aware and bounded: +- Virtualenv layout roots are detected via `pyvenv.cfg` or `venv/`-style directories. +- Site-packages roots include `lib/python*/site-packages` and `lib/python*/dist-packages`. +- Container unpack layouts are supported as additional candidate roots: + - `layers/*` (direct children) + - `.layers/*` (direct children) + - `layer*` (direct children of the analysis root) + +## Virtual filesystem (VFS) and determinism +- Inputs are normalized deterministically (dedupe + stable ordering); later/higher-confidence inputs override earlier ones in the VFS overlay. +- Archive virtual roots are stable and collision-safe: + - `archives/wheel/` + - `archives/zipapp/` + - `archives/sdist/` + - collisions use a deterministic `~N` suffix +- Evidence locators are always analysis-root relative and use `/` separators. + +## Identity rules (PURL vs explicit key) +Concrete versions emit a PURL: +- `purl = pkg:pypi/@` + +Non-concrete declarations emit an explicit key: +- `componentKey = explicit::::pypi::::sha256:` +- `purl = null`, `version = null` +- generated via `LanguageExplicitKey.Create(...)` and aligned with `docs/modules/scanner/language-analyzers-contract.md` + +Editable declarations (from requirements `--editable` / `-e`) normalize the specifier: +- project-relative paths stay relative (`editable-src`) +- absolute/host paths are redacted and never appear in the digest input + +## Evidence and metadata +Installed and archive distributions emit evidence for (when present): +- `METADATA`, `RECORD`, `WHEEL`, `INSTALLER`, `entry_points.txt`, `direct_url.json` + +`RECORD` verification emits deterministic counters: +- `record.totalEntries`, `record.hashedEntries`, `record.missingFiles`, `record.hashMismatches`, `record.ioErrors` +- plus `record.unsupportedAlgorithms` when algorithms outside the supported set are present + +Declared-only/lock-only components include: +- `declaredOnly=true` +- `lockSource`, `lockLocator`, optional `lockResolved`, `lockIndex`, `lockExtras`, `lockEditablePath` + +## Container overlay semantics (pending contract) +When scanning raw OCI layer trees, correct overlay/whiteout handling is contract-driven. Until that contract lands, treat per-layer inventory as best-effort and do not rely on it as a merged-rootfs truth source. + +## Vendored/bundled packages (pending contract) +Vendored directory signals are detected but representation (separate components vs parent-only metadata) is contract-driven to avoid false vulnerability joins. + +## References +- Sprint: `docs/implplan/SPRINT_0405_0001_0001_scanner_python_detection_gaps.md` +- Cross-analyzer contract: `docs/modules/scanner/language-analyzers-contract.md` +- Implementation: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/PythonLanguageAnalyzer.cs` + diff --git a/docs/modules/scanner/architecture.md b/docs/modules/scanner/architecture.md index 0f191c90a..3b22549d1 100644 --- a/docs/modules/scanner/architecture.md +++ b/docs/modules/scanner/architecture.md @@ -42,9 +42,14 @@ src/ └─ Tools/ ├─ StellaOps.Scanner.Sbomer.BuildXPlugin/ # BuildKit generator (image referrer SBOMs) └─ StellaOps.Scanner.Sbomer.DockerImage/ # CLI‑driven scanner container -``` - -Analyzer assemblies and buildx generators are packaged as **restart-time plug-ins** under `plugins/scanner/**` with manifests; services must restart to activate new plug-ins. +``` + +Per-analyzer notes (language analyzers): +- `docs/modules/scanner/analyzers-java.md` +- `docs/modules/scanner/analyzers-bun.md` +- `docs/modules/scanner/analyzers-python.md` + +Analyzer assemblies and buildx generators are packaged as **restart-time plug-ins** under `plugins/scanner/**` with manifests; services must restart to activate new plug-ins. ### 1.2 Native reachability upgrades (Nov 2026) @@ -397,7 +402,9 @@ scanner: --- -## 12) Testing matrix +## 12) Testing matrix + +* **Analyzer contracts:** see `language-analyzers-contract.md` and per-analyzer docs (e.g., `analyzers-java.md`, Sprint 0403). * **Determinism:** given same image + analyzers → byte‑identical **CDX Protobuf**; JSON normalized. * **OS packages:** ground‑truth images per distro; compare to package DB. diff --git a/docs/modules/scanner/language-analyzers-contract.md b/docs/modules/scanner/language-analyzers-contract.md new file mode 100644 index 000000000..323c34ea9 --- /dev/null +++ b/docs/modules/scanner/language-analyzers-contract.md @@ -0,0 +1,110 @@ +# Scanner Language Analyzer Contracts (Identity / Evidence / Container Layout) + +This document freezes the cross-analyzer contracts that are shared by the language analyzers (Java, .NET, Python, Node, Bun). These rules exist to prevent false matches, keep outputs deterministic, and protect against host-path leakage. + +## 1) Identity Safety Contract (PURL vs Explicit Key) + +### 1.1 Goals +- **No fake versions**: never encode version ranges, tags, local paths, or git URLs as a versioned PURL. +- **No collisions**: explicit-key identities must not collide with concrete PURLs and must be deterministic across OS path separators. +- **Proof-first**: emit concrete PURLs only when the analyzer has concrete, replayable evidence for the version. + +### 1.2 When to emit a concrete PURL +Emit a concrete (versioned) PURL only when **both** are true: +1) The analyzer can determine a **concrete version** (ecosystem-specific) for the component. +2) The version is backed by **replayable evidence** (e.g., installed artifact metadata or lockfile-resolved entry). + +Typical sources that qualify: +- **Installed inventory** (e.g., `node_modules/**/package.json`, Python `*.dist-info/METADATA`, .NET `deps.json` entries). +- **Lockfile-resolved inventory** (e.g., `bun.lock` entry with `name@version` and integrity/resolved URL). + +### 1.3 When to emit an explicit-key component (required) +Emit an explicit-key component when the dependency is **declared-only** or otherwise **non-concrete**: +- Version ranges / operators (`^`, `~`, `>=`, `<`, `*`, `x`, `latest`, etc.). +- Workspace/link/file dependencies (`workspace:*`, `link:`, `file:`, local path refs, editable installs). +- Git dependencies (git URL / commit / ref) when a concrete semantic version is not provable from local evidence. +- Unknown / missing version. + +**Rule:** If the analyzer cannot prove a concrete version from local evidence, it must not emit a versioned PURL for that dependency. + +### 1.4 Explicit-key format (canonical) +For declared-only / non-concrete identities, analyzers must emit: +- `componentKey`: `explicit::::::::sha256:` +- `purl`: `null` +- `version`: `null` + +Where `` is `sha256` of the canonical UTF-8 string: +``` +\n\n\n +``` + +Canonicalization rules: +- `` uses ecosystem naming rules (e.g., npm scoped names keep `@scope/name`). +- `` is the **original declared specifier** (range/tag/url/path), trimmed; for unknown, use `""`. +- `` is project-relative with `/` separators (e.g., `package.json#dependencies`, `requirements.txt`, `Directory.Packages.props#PackageVersion:Foo`). +- No absolute paths, drive letters, or host roots appear in any input to the digest. + +### 1.5 Required metadata for explicit-key components +Explicit-key components must include (at minimum) these metadata keys: +- `declaredOnly=true` +- `declared.source=` (e.g., `package.json`, `Directory.Packages.props`) +- `declared.locator=` (same string used in digest) +- `declared.versionSpec=` (original specifier or empty) +- `declared.scope=` when applicable +- `declared.sourceType=` + +## 2) Evidence Locator Contract + +### 2.1 General rules +- Evidence locators are **external-facing** and must be stable and parseable. +- Every locator is **project-relative** with `/` separators (never absolute). +- Evidence content/hashing must be bounded; when bounds are exceeded, emit deterministic `skipped` markers in metadata instead of silently omitting. + +### 2.2 Locator formats (canonical) +**File evidence** +- `locator`: `` (e.g., `packages/app/package.json`) +- `source`: a stable discriminator (e.g., `package.json`, `pom.xml`, `METADATA`) + +**Lockfile entry evidence** +- `locator`: `:` +- Examples: + - Node package-lock: `package-lock.json:packages/app/node_modules/foo` + - Bun lock: `bun.lock:packages[foo@1.2.3]` + - Maven/Gradle lock: `gradle.lockfile:com.example:foo:1.2.3` + +**Nested artifact evidence** +- `locator`: `!!` +- Example: `demo-jni.jar!META-INF/native-image/demo/jni-config.json` + +**Derived evidence** +- `locator`: a stable synthetic name (e.g., `phase22.ndjson`) +- `source`: a stable synthetic source (e.g., `node.observation`) + +### 2.3 Hashing rules (baseline) +- Hash only bounded inputs (default: 1 MiB per evidence value/file; analyzers may choose a tighter cap). +- Hash algorithm: `sha256` over UTF-8 bytes for textual evidence, raw bytes for file evidence. +- If hashing is skipped due to bounds or errors, emit deterministic metadata markers (e.g., `hashSkipped=true`, `hashSkipped.reason=sizeCap`). + +## 3) Container Layout Discovery Contract + +### 3.1 Layer root candidates +Language analyzers that support container-root discovery must treat these as **candidate roots** under the analysis root: +- `layers/*` (direct children) +- `.layers/*` (direct children; **must not be skipped**) +- `layer*` (direct children of the analysis root, e.g., `layer1/`, `layer2/`) + +Each candidate root is scanned independently for projects. + +### 3.2 Bounds and traversal safety (required) +- Deterministic traversal (sorted directory enumeration). +- Depth caps per candidate root; hard cap on total discovered project roots. +- Must never recurse into `node_modules/` (Node/Bun) or equivalent heavy dirs. +- Hidden directories may be skipped **except** `.layers` which is treated as a top-level candidate root. +- No symlink escape: if symlinks are followed, resolved targets must remain within the candidate root prefix and cycles must be prevented. + +### 3.3 Overlay/whiteout semantics +- If an analyzer implements overlay semantics (notably Python container adapters), whiteouts and precedence rules must be explicit, deterministic, and fixture-tested. +- If an analyzer does **not** implement overlay semantics, it must still keep discovery bounded and must not silently drop projects; emit deterministic "skipped" markers when bounds prevent full traversal. + +## Compliance +Sprints `docs/implplan/SPRINT_0403_0001_0001_scanner_java_detection_gaps.md` through `docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md` (and the program sprint `docs/implplan/SPRINT_0408_0001_0001_scanner_language_detection_gaps_program.md`) carry the per-analyzer implementation and test evidence required to enforce this contract. diff --git a/docs/policy/dsl.md b/docs/policy/dsl.md index a44790d83..54173f3f6 100644 --- a/docs/policy/dsl.md +++ b/docs/policy/dsl.md @@ -1,7 +1,7 @@ # Stella Policy DSL (`stella-dsl@1`) -> **Audience:** Policy authors, reviewers, and tooling engineers building lint/compile flows for the Policy Engine v2 rollout (Sprint 20). -> **Imposed rule:** Policies that alter reachability or trust weighting must run in shadow mode first with coverage fixtures; promotion to active is blocked until shadow + coverage gates pass. +> **Audience:** Policy authors, reviewers, and tooling engineers building lint/compile flows for the Policy Engine v2 rollout (Sprint 20). +> **Imposed rule:** Policies that alter reachability or trust weighting must run in shadow mode first with coverage fixtures; promotion to active is blocked until shadow + coverage gates pass. This document specifies the `stella-dsl@1` grammar, semantics, and guardrails used by Stella Ops to transform SBOM facts, Concelier advisories, and Excititor VEX statements into effective findings. Use it with the [Policy Engine Overview](overview.md) for architectural context and the upcoming lifecycle/run guides for operational workflows. @@ -9,13 +9,13 @@ This document specifies the `stella-dsl@1` grammar, semantics, and guardrails us ## 1 · Design Goals -- **Deterministic:** Same policy + same inputs ⇒ identical findings on every machine. -- **Declarative:** No arbitrary loops, network calls, or clock access. -- **Explainable:** Every decision records the rule, inputs, and rationale in the explain trace. -- **Lean authoring:** Common precedence, severity, and suppression patterns are first-class. -- **Offline-friendly:** Grammar and built-ins avoid cloud dependencies, run the same in sealed deployments. -- **Reachability-aware:** Policies can consume reachability lattice states (`ReachState`) and evidence scores to drive VEX gates (`not_affected`, `under_investigation`, `affected`). -- **Signal-first:** Trust, reachability, entropy, and uncertainty signals are first-class so explain traces stay reproducible. +- **Deterministic:** Same policy + same inputs ⇒ identical findings on every machine. +- **Declarative:** No arbitrary loops, network calls, or clock access. +- **Explainable:** Every decision records the rule, inputs, and rationale in the explain trace. +- **Lean authoring:** Common precedence, severity, and suppression patterns are first-class. +- **Offline-friendly:** Grammar and built-ins avoid cloud dependencies, run the same in sealed deployments. +- **Reachability-aware:** Policies can consume reachability lattice states (`ReachState`) and evidence scores to drive VEX gates (`not_affected`, `under_investigation`, `affected`). +- **Signal-first:** Trust, reachability, entropy, and uncertainty signals are first-class so explain traces stay reproducible. --- @@ -42,26 +42,26 @@ policy "Default Org Policy" syntax "stella-dsl@1" { } } - rule vex_precedence priority 10 { - when vex.any(status in ["not_affected","fixed"]) - and vex.justification in ["component_not_present","vulnerable_code_not_present"] - then status := vex.status - because "Strong vendor justification prevails"; - } - - rule reachability_gate priority 20 { - when telemetry.reachability.state == "reachable" and telemetry.reachability.score >= 0.6 - then status := "affected" - because "Runtime/graph evidence shows reachable code path"; - } - - rule trust_penalty priority 30 { - when signals.trust_score < 0.4 or signals.entropy_penalty > 0.2 - then severity := severity_band("critical") - because "Low trust score or high entropy"; - } -} -``` + rule vex_precedence priority 10 { + when vex.any(status in ["not_affected","fixed"]) + and vex.justification in ["component_not_present","vulnerable_code_not_present"] + then status := vex.status + because "Strong vendor justification prevails"; + } + + rule reachability_gate priority 20 { + when telemetry.reachability.state == "reachable" and telemetry.reachability.score >= 0.6 + then status := "affected" + because "Runtime/graph evidence shows reachable code path"; + } + + rule trust_penalty priority 30 { + when signals.trust_score < 0.4 or signals.entropy_penalty > 0.2 + then severity := severity_band("critical") + because "Low trust score or high entropy"; + } +} +``` High-level layout: @@ -141,10 +141,10 @@ annotate = "annotate", identifier, ":=", expression, ";" ; Notes: -- `helper` is reserved for shared calculcations (not yet implemented in `@1`). -- `else` branch executes only if `when` predicates evaluate truthy **and** no prior rule earlier in priority handled the tuple. -- Semicolons inside rule bodies are optional when each clause is on its own line; the compiler emits canonical semicolons in IR. -- `settings.shadow = true` enables shadow-mode evaluation (findings recorded but not enforced). Promotion gates require at least one shadow run with coverage fixtures. +- `helper` is reserved for shared calculcations (not yet implemented in `@1`). +- `else` branch executes only if `when` predicates evaluate truthy **and** no prior rule earlier in priority handled the tuple. +- Semicolons inside rule bodies are optional when each clause is on its own line; the compiler emits canonical semicolons in IR. +- `settings.shadow = true` enables shadow-mode evaluation (findings recorded but not enforced). Promotion gates require at least one shadow run with coverage fixtures. --- @@ -152,23 +152,23 @@ Notes: Within predicates and actions you may reference the following namespaces: -| Namespace | Fields | Description | -|-----------|--------|-------------| -| `sbom` | `purl`, `name`, `version`, `licenses`, `layerDigest`, `tags`, `usedByEntrypoint` | Component metadata from Scanner. | -| `advisory` | `id`, `source`, `aliases`, `severity`, `cvss`, `publishedAt`, `modifiedAt`, `content.raw` | Canonical Concelier advisory view. | -| `vex` | `status`, `justification`, `statementId`, `timestamp`, `scope` | Current VEX statement when iterating; aggregator helpers available. | -| `vex.any(...)`, `vex.all(...)`, `vex.count(...)` | Functions operating over all matching statements. | -| `run` | `policyId`, `policyVersion`, `tenant`, `timestamp` | Metadata for explain annotations. | -| `env` | Arbitrary key/value pairs injected per run (e.g., `environment`, `runtime`). | -| `telemetry` | Optional reachability signals. Example fields: `telemetry.reachability.state`, `telemetry.reachability.score`, `telemetry.reachability.policyVersion`. Missing fields evaluate to `unknown`. | -| `signals` | Normalised signal dictionary: `trust_score` (0–1), `reachability.state` (`reachable|unreachable|unknown|under_investigation`), `reachability.score` (0–1), `reachability.confidence` (0–1), `reachability.evidence_ref` (string), `entropy_penalty` (0–0.3), `uncertainty.level` (`U1`–`U3`), `runtime_hits` (bool). | -| `secret` | `findings`, `bundle`, helper predicates | Populated when the Secrets Analyzer runs. Exposes masked leak findings and bundle metadata for policy decisions. | -| `profile.` | Values computed inside profile blocks (maps, scalars). | - -> **Reachability evidence gate.** When `reachability.state == "unreachable"` but `reachability.evidence_ref` is missing (or confidence is below the high-confidence threshold), Policy Engine downgrades the state to `under_investigation` to avoid false "not affected" claims. -> -> **Secrets namespace.** When `StellaOps.Scanner.Analyzers.Secrets` is enabled the Policy Engine receives masked findings (`secret.findings[*]`) plus bundle metadata (`secret.bundle.id`, `secret.bundle.version`). Policies should rely on the helper predicates listed below rather than reading raw arrays to preserve determinism and future compatibility. - +| Namespace | Fields | Description | +|-----------|--------|-------------| +| `sbom` | `purl`, `name`, `version`, `licenses`, `layerDigest`, `tags`, `usedByEntrypoint` | Component metadata from Scanner. | +| `advisory` | `id`, `source`, `aliases`, `severity`, `cvss`, `publishedAt`, `modifiedAt`, `content.raw` | Canonical Concelier advisory view. | +| `vex` | `status`, `justification`, `statementId`, `timestamp`, `scope` | Current VEX statement when iterating; aggregator helpers available. | +| `vex.any(...)`, `vex.all(...)`, `vex.count(...)` | Functions operating over all matching statements. | +| `run` | `policyId`, `policyVersion`, `tenant`, `timestamp` | Metadata for explain annotations. | +| `env` | Arbitrary key/value pairs injected per run (e.g., `environment`, `runtime`). | +| `telemetry` | Optional reachability signals. Example fields: `telemetry.reachability.state`, `telemetry.reachability.score`, `telemetry.reachability.policyVersion`. Missing fields evaluate to `unknown`. | +| `signals` | Normalised signal dictionary: `trust_score` (0–1), `reachability.state` (`reachable|unreachable|unknown|under_investigation`), `reachability.score` (0–1), `reachability.confidence` (0–1), `reachability.evidence_ref` (string), `entropy_penalty` (0–0.3), `uncertainty.level` (`U1`–`U3`), `runtime_hits` (bool). | +| `secret` | `findings`, `bundle`, helper predicates | Populated when the Secrets Analyzer runs. Exposes masked leak findings and bundle metadata for policy decisions. | +| `profile.` | Values computed inside profile blocks (maps, scalars). | + +> **Reachability evidence gate.** When `reachability.state == "unreachable"` but `reachability.evidence_ref` is missing (or confidence is below the high-confidence threshold), Policy Engine downgrades the state to `under_investigation` to avoid false "not affected" claims. +> +> **Secrets namespace.** When `StellaOps.Scanner.Analyzers.Secrets` is enabled the Policy Engine receives masked findings (`secret.findings[*]`) plus bundle metadata (`secret.bundle.id`, `secret.bundle.version`). Policies should rely on the helper predicates listed below rather than reading raw arrays to preserve determinism and future compatibility. + Missing fields evaluate to `null`, which is falsey in boolean context and propagates through comparisons unless explicitly checked. --- @@ -180,50 +180,50 @@ Missing fields evaluate to `null`, which is falsey in boolean context and propag | `normalize_cvss(advisory)` | `Advisory → SeverityScalar` | Parses `advisory.content.raw` for CVSS data; falls back to policy maps. | | `cvss(score, vector)` | `double × string → SeverityScalar` | Constructs a severity object manually. | | `severity_band(value)` | `string → SeverityBand` | Normalises strings like `"critical"`, `"medium"`. | -| `risk_score(base, modifiers...)` | Variadic | Multiplies numeric modifiers (severity × trust × reachability). | -| `reach_state(state)` | `string → ReachState` | Normalises reachability state strings (`reachable`, `unreachable`, `unknown`, `under_investigation`). | -| `vex.any(predicate)` | `(Statement → bool) → bool` | `true` if any statement satisfies predicate. | +| `risk_score(base, modifiers...)` | Variadic | Multiplies numeric modifiers (severity × trust × reachability). | +| `reach_state(state)` | `string → ReachState` | Normalises reachability state strings (`reachable`, `unreachable`, `unknown`, `under_investigation`). | +| `vex.any(predicate)` | `(Statement → bool) → bool` | `true` if any statement satisfies predicate. | | `vex.all(predicate)` | `(Statement → bool) → bool` | `true` if all statements satisfy predicate. | | `vex.latest()` | `→ Statement` | Lexicographically newest statement. | | `advisory.has_tag(tag)` | `string → bool` | Checks advisory metadata tags. | | `advisory.matches(pattern)` | `string → bool` | Glob match against advisory identifiers. | -| `sbom.has_tag(tag)` | `string → bool` | Uses SBOM inventory tags (usage vs inventory). | -| `sbom.any_component(predicate)` | `(Component → bool) → bool` | Iterates SBOM components, exposing `component` plus language scopes (e.g., `ruby`). | +| `sbom.has_tag(tag)` | `string → bool` | Uses SBOM inventory tags (usage vs inventory). | +| `sbom.any_component(predicate)` | `(Component → bool) → bool` | Iterates SBOM components, exposing `component` plus language scopes (e.g., `ruby`). | | `exists(expression)` | `→ bool` | `true` when value is non-null/empty. | -| `coalesce(a, b, ...)` | `→ value` | First non-null argument. | -| `days_between(dateA, dateB)` | `→ int` | Absolute day difference (UTC). | -| `percent_of(part, whole)` | `→ double` | Fractions for scoring adjustments. | -| `lowercase(text)` | `string → string` | Normalises casing deterministically (InvariantCulture). | -| `secret.hasFinding(ruleId?, severity?, confidence?)` | `→ bool` | True if any secret leak finding matches optional filters. | -| `secret.match.count(ruleId?)` | `→ int` | Count of findings, optionally scoped to a rule ID. | -| `secret.bundle.version(required)` | `string → bool` | Ensures the active secret rule bundle version ≥ required (semantic compare). | -| `secret.mask.applied` | `→ bool` | Indicates whether masking succeeded for all surfaced payloads. | -| `secret.path.allowlist(patterns)` | `list → bool` | True when all findings fall within allowed path patterns (useful for waivers). | - -All built-ins are pure; if inputs are null the result is null unless otherwise noted. - ---- - -### 6.1 · Ruby Component Scope - -Inside `sbom.any_component(...)`, Ruby gems surface a `ruby` scope with the following helpers: - -| Helper | Signature | Description | -|--------|-----------|-------------| -| `ruby.group(name)` | `string → bool` | Matches Bundler group membership (`development`, `test`, etc.). | -| `ruby.groups()` | `→ set` | Returns all groups for the active component. | -| `ruby.declared_only()` | `→ bool` | `true` when no vendor cache artefacts were observed for the gem. | -| `ruby.source(kind?)` | `string? → bool` | Returns the raw source when called without args, or matches provenance kinds (`registry`, `git`, `path`, `vendor-cache`). | -| `ruby.capability(name)` | `string → bool` | Checks capability flags emitted by the analyzer (`exec`, `net`, `scheduler`, `scheduler.activejob`, etc.). | -| `ruby.capability_any(names)` | `set → bool` | `true` when any capability in the set is present. | - -Scheduler capability sub-types use dot notation (`ruby.capability("scheduler.sidekiq")`) and inherit from the broad `scheduler` capability. - ---- - -## 7 · Rule Semantics - -1. **Ordering:** Rules execute in ascending `priority`. When priorities tie, lexical order defines precedence. +| `coalesce(a, b, ...)` | `→ value` | First non-null argument. | +| `days_between(dateA, dateB)` | `→ int` | Absolute day difference (UTC). | +| `percent_of(part, whole)` | `→ double` | Fractions for scoring adjustments. | +| `lowercase(text)` | `string → string` | Normalises casing deterministically (InvariantCulture). | +| `secret.hasFinding(ruleId?, severity?, confidence?)` | `→ bool` | True if any secret leak finding matches optional filters. | +| `secret.match.count(ruleId?)` | `→ int` | Count of findings, optionally scoped to a rule ID. | +| `secret.bundle.version(required)` | `string → bool` | Ensures the active secret rule bundle version ≥ required (semantic compare). | +| `secret.mask.applied` | `→ bool` | Indicates whether masking succeeded for all surfaced payloads. | +| `secret.path.allowlist(patterns)` | `list → bool` | True when all findings fall within allowed path patterns (useful for waivers). | + +All built-ins are pure; if inputs are null the result is null unless otherwise noted. + +--- + +### 6.1 · Ruby Component Scope + +Inside `sbom.any_component(...)`, Ruby gems surface a `ruby` scope with the following helpers: + +| Helper | Signature | Description | +|--------|-----------|-------------| +| `ruby.group(name)` | `string → bool` | Matches Bundler group membership (`development`, `test`, etc.). | +| `ruby.groups()` | `→ set` | Returns all groups for the active component. | +| `ruby.declared_only()` | `→ bool` | `true` when no vendor cache artefacts were observed for the gem. | +| `ruby.source(kind?)` | `string? → bool` | Returns the raw source when called without args, or matches provenance kinds (`registry`, `git`, `path`, `vendor-cache`). | +| `ruby.capability(name)` | `string → bool` | Checks capability flags emitted by the analyzer (`exec`, `net`, `scheduler`, `scheduler.activejob`, etc.). | +| `ruby.capability_any(names)` | `set → bool` | `true` when any capability in the set is present. | + +Scheduler capability sub-types use dot notation (`ruby.capability("scheduler.sidekiq")`) and inherit from the broad `scheduler` capability. + +--- + +## 7 · Rule Semantics + +1. **Ordering:** Rules execute in ascending `priority`. When priorities tie, lexical order defines precedence. 2. **Short-circuit:** Once a rule sets `status`, subsequent rules only execute if they use `combine`. Use this sparingly to avoid ambiguity. 3. **Actions:** - `status := ` – Allowed values: `affected`, `not_affected`, `fixed`, `suppressed`, `under_investigation`, `escalated`. @@ -271,30 +271,30 @@ rule vex_strong_claim priority 5 { } ``` -### 9.3 Environment-Specific Escalation +### 9.3 Environment-Specific Escalation ```dsl -rule internet_exposed_guard { - when env.exposure == "internet" - and severity.normalized >= "High" - then escalate to severity_band("Critical") - because "Internet-exposed assets require critical posture"; -} -``` - -### 9.4 Shadow mode & coverage - -- Enable `settings { shadow = true; }` for new policies or major changes. Findings are recorded but not enforced. -- Provide coverage fixtures under `tests/policy//cases/*.json`; run `stella policy test` locally and in CI. Coverage results must be attached on submission. -- Promotion to active is blocked until shadow runs + coverage gates pass (see lifecycle §3). - -### 9.5 Authoring workflow (quick checklist) - -1. Write/update policy with shadow enabled. -2. Add/refresh coverage fixtures; run `stella policy test`. -3. `stella policy lint` and `stella policy simulate --fixtures ...` with expected signals (trust_score, reachability, entropy_penalty) noted in comments. -4. Submit with attachments: lint, simulate diff, coverage results. -5. After approval, disable shadow and promote; retain fixtures for regression tests. +rule internet_exposed_guard { + when env.exposure == "internet" + and severity.normalized >= "High" + then escalate to severity_band("Critical") + because "Internet-exposed assets require critical posture"; +} +``` + +### 9.4 Shadow mode & coverage + +- Enable `settings { shadow = true; }` for new policies or major changes. Findings are recorded but not enforced. +- Provide coverage fixtures under `tests/policy//cases/*.json`; run `stella policy test` locally and in CI. Coverage results must be attached on submission. +- Promotion to active is blocked until shadow runs + coverage gates pass (see lifecycle §3). + +### 9.5 Authoring workflow (quick checklist) + +1. Write/update policy with shadow enabled. +2. Add/refresh coverage fixtures; run `stella policy test`. +3. `stella policy lint` and `stella policy simulate --fixtures ...` with expected signals (trust_score, reachability, entropy_penalty) noted in comments. +4. Submit with attachments: lint, simulate diff, coverage results. +5. After approval, disable shadow and promote; retain fixtures for regression tests. ### 9.4 Anti-pattern (flagged by linter) @@ -332,7 +332,42 @@ rule catch_all { --- -## 12 · Versioning & Compatibility +## 12 · Uncertainty Gates (U1/U2/U3) + +Uncertainty gates enforce evidence-quality thresholds before allowing high-confidence VEX decisions. When entropy is too high or evidence is missing, policies should downgrade to \ rather than risk false negatives. + +### 12.1 Gate Types + +| Gate | Tier Threshold | Blocks | Allows | Remediation | +|------|---------------|--------|--------|-------------| +| \ | T1 (\) | \ | \, \ | Upload symbols, resolve unknowns | +| \ | T2 (\) | \ (warns) | \ with review flag | Populate lockfiles, fix purl resolution | +| \ | T3 (\) | None (advisory only) | All with caveat | Corroborate advisory, add trusted source | + +### 12.2 Uncertainty Gate Rules + +### 12.3 Tier-Aware Compound Rules + +Combine uncertainty tiers with reachability states for nuanced gating: + +### 12.4 Remediation Actions + +Policy rules should guide users toward reducing uncertainty: + +| Uncertainty State | Remediation Action | Policy Annotation | +|-------------------|-------------------|-------------------| +| \ (MissingSymbolResolution) | Upload debug symbols, run \ | \ | +| \ (MissingPurl) | Generate lockfiles, verify package coordinates | \ | +| \ (UntrustedAdvisory) | Cross-reference trusted sources, wait for corroboration | \ | +| \ (Unknown) | Run initial analysis, enable probes | \ | + +### 12.5 YAML Configuration for Gate Thresholds + +The Policy Engine reads uncertainty gate thresholds from configuration: + +--- + +## 13 · Versioning & Compatibility - `syntax "stella-dsl@1"` is mandatory. - Future revisions (`@2`, …) will be additive; existing packs continue to compile with their declared version. @@ -340,7 +375,7 @@ rule catch_all { --- -## 13 · Compliance Checklist +## 14 · Compliance Checklist - [ ] **Grammar validated:** Policy compiles with `stella policy lint` and matches `syntax "stella-dsl@1"`. - [ ] **Deterministic constructs only:** No use of forbidden namespaces (`DateTime.Now`, `Guid.NewGuid`, external services). @@ -351,4 +386,4 @@ rule catch_all { --- -*Last updated: 2025-11-26 (Sprint 0401).* +*Last updated: 2025-12-13 (Sprint 0401).* diff --git a/docs/reachability/binary-reachability-schema.md b/docs/reachability/binary-reachability-schema.md new file mode 100644 index 000000000..1c04bad4a --- /dev/null +++ b/docs/reachability/binary-reachability-schema.md @@ -0,0 +1,461 @@ +# Binary Reachability Schema + +_Last updated: 2025-12-13. Owner: Scanner Guild + Attestor Guild._ + +This document defines the binary reachability schema addressing gaps BR1-BR10 from the November 2025 product findings. It specifies DSSE predicate formats, edge hash recipes, binary evidence requirements, build-id handling, and Sigstore integration. + +--- + +## 1. Overview + +Binary reachability extends the function-level evidence chain to native executables (ELF, PE, Mach-O). Key challenges addressed: + +- **Stripped binaries:** Symbol recovery using `code_id` + `code_block_hash` +- **Build variants:** Handling multiple builds from same source +- **Large graphs:** Chunking and size limits for DSSE/Rekor +- **Offline verification:** Air-gapped attestation workflows + +--- + +## 2. Gap Resolutions + +### BR1: Canonical DSSE/Predicate Schemas + +**Binary graph predicate:** + +``` +stella.ops/binaryGraph@v1 +``` + +**Predicate schema:** + +```json +{ + "_type": "https://stellaops.dev/predicates/binaryGraph/v1", + "subject": [ + { + "name": "graph", + "digest": {"blake3": "a1b2c3d4e5f6..."} + } + ], + "predicate": { + "analyzer": { + "name": "scanner.native", + "version": "1.2.0", + "toolchain": "ghidra-11.2" + }, + "binary": { + "format": "ELF", + "arch": "x86_64", + "file_hash": "sha256:...", + "build_id": "gnu-build-id:5f0c7c3c..." + }, + "graph_stats": { + "node_count": 1247, + "edge_count": 3891, + "root_count": 5 + }, + "evidence": { + "symbols_source": "DWARF", + "stripped_symbols": 58, + "heuristic_symbols": 12 + }, + "created_at": "2025-12-13T10:00:00Z" + } +} +``` + +**Edge bundle predicate:** + +``` +stella.ops/binaryEdgeBundle@v1 +``` + +```json +{ + "_type": "https://stellaops.dev/predicates/binaryEdgeBundle/v1", + "subject": [ + { + "name": "edges", + "digest": {"sha256": "..."} + } + ], + "predicate": { + "graph_hash": "blake3:a1b2c3d4...", + "bundle_id": "bundle:001", + "bundle_reason": "init_array", + "edge_count": 128, + "edges": [ + { + "from": "sym:binary:...", + "to": "sym:binary:...", + "reason": "init-array", + "confidence": 0.95 + } + ] + } +} +``` + +### BR2: Edge Hash Recipe + +**Binary edge hash computation:** + +``` +edge_id = "edge:" + sha256( + canonical_json({ + "from": edge.from, + "to": edge.to, + "kind": edge.kind, + "reason": edge.reason, + "binary_hash": binary.file_hash // Binary context included + }) +) +``` + +**Hash includes binary context:** + +Unlike managed code edges, binary edges include `binary_hash` in the hash computation to distinguish edges from different binaries with identical symbol names. + +**Canonicalization:** + +1. Keys: `binary_hash`, `from`, `kind`, `reason`, `to` (alphabetical) +2. No whitespace, UTF-8 encoding +3. Lowercase hex for all hashes + +### BR3: Required Binary Evidence with CAS Refs + +**Required evidence per node:** + +| Evidence Type | Required | CAS Storage | +|---------------|----------|-------------| +| File hash | Yes | N/A (inline) | +| Build ID | Conditional | N/A (inline) | +| Symbol source | Yes | N/A (inline) | +| Code block hash | For stripped | `cas://binary/blocks/{sha256}` | +| Disassembly | Optional | `cas://binary/disasm/{sha256}` | +| CFG | Optional | `cas://binary/cfg/{sha256}` | + +**Evidence schema:** + +```json +{ + "binary_evidence": { + "file_hash": "sha256:...", + "build_id": "gnu-build-id:5f0c7c3c...", + "symbol_source": "DWARF", + "symbol_confidence": 0.95, + "code_block_hash": "sha256:deadbeef...", + "code_block_uri": "cas://binary/blocks/sha256:deadbeef...", + "disassembly_uri": "cas://binary/disasm/sha256:...", + "cfg_uri": "cas://binary/cfg/sha256:..." + } +} +``` + +**CAS layout:** + +``` +cas://binary/ + blocks/{sha256}/ # Code block bytes + disasm/{sha256}/ # Disassembly JSON + cfg/{sha256}/ # Control flow graph + symbols/{sha256}/ # Symbol table extract +``` + +### BR4: Build-ID/Variant Rules + +**Build-ID sources:** + +| Format | Build-ID Source | Example | +|--------|-----------------|---------| +| ELF | `.note.gnu.build-id` | `gnu-build-id:5f0c7c3c...` | +| PE | Debug GUID | `pe-guid:12345678-1234-...` | +| Mach-O | `LC_UUID` | `macho-uuid:12345678...` | + +**Fallback when build-ID absent:** + +```json +{ + "build_id": null, + "build_id_fallback": { + "method": "file_hash", + "value": "sha256:...", + "confidence": 0.7 + } +} +``` + +**Variant handling:** + +Multiple binaries from same source (debug/release, different arch): + +```json +{ + "variant_group": "sha256:source_hash...", + "variants": [ + {"build_id": "gnu-build-id:aaa...", "variant_type": "release-x86_64"}, + {"build_id": "gnu-build-id:bbb...", "variant_type": "debug-x86_64"}, + {"build_id": "gnu-build-id:ccc...", "variant_type": "release-aarch64"} + ] +} +``` + +### BR5: Policy Hash Governance + +**Policy version binding:** + +Binary reachability graphs are bound to a policy version: + +```json +{ + "policy_binding": { + "policy_digest": "sha256:...", + "policy_version": "P-7:v4", + "bound_at": "2025-12-13T10:00:00Z", + "binding_mode": "strict" + } +} +``` + +**Binding modes:** + +| Mode | Behavior | +|------|----------| +| `strict` | Graph invalid if policy changes | +| `forward` | Graph valid with newer policy versions | +| `any` | Graph valid with any policy version | + +**Governance rules:** + +1. Production graphs use `strict` binding +2. Test graphs may use `forward` +3. Policy hash computed from canonical DSL +4. Binding stored in graph metadata + +### BR6: Sigstore Bundle/Log Routing + +**Sigstore integration:** + +```json +{ + "sigstore": { + "bundle_type": "hashedrekord", + "log_index": 12345678, + "log_id": "rekor.sigstore.dev", + "inclusion_proof": { + "log_index": 12345678, + "root_hash": "sha256:...", + "tree_size": 98765432, + "hashes": ["sha256:...", "sha256:..."] + }, + "signed_entry_timestamp": "base64:..." + } +} +``` + +**Log routing:** + +| Evidence Type | Log | Notes | +|---------------|-----|-------| +| Graph DSSE | Rekor (public) | Always | +| Edge bundle DSSE | Rekor (capped) | Configurable limit | +| Code block | No log | CAS only | +| CFG/Disasm | No log | CAS only | + +**Offline mode:** + +When Rekor unavailable: + +```json +{ + "sigstore": { + "mode": "offline", + "checkpoint": { + "origin": "rekor.sigstore.dev", + "checkpoint_data": "base64:...", + "captured_at": "2025-12-13T10:00:00Z" + }, + "deferred_submission": true + } +} +``` + +### BR7: Idempotent Submission Keys + +**Submission key format:** + +``` +submit:{tenant}:{binary_hash}:{graph_hash}:{timestamp_hour} +``` + +**Idempotency rules:** + +1. Same key returns existing entry (no duplicate) +2. Key includes hour-granularity timestamp for rate limiting +3. Different graphs from same binary produce different keys +4. Retry within 1 hour uses same key + +**Implementation:** + +```json +{ + "submission": { + "key": "submit:acme:sha256:abc...:blake3:def...:2025121310", + "status": "accepted", + "existing_entry": false, + "log_index": 12345678 + } +} +``` + +### BR8: Size/Chunking Limits + +**Size limits:** + +| Element | Limit | Action on Exceed | +|---------|-------|------------------| +| Graph JSON | 10 MB | Chunk nodes/edges | +| Edge bundle | 512 edges | Split bundles | +| DSSE payload | 1 MB | Compress/chunk | +| Rekor entry | 100 KB | Reference CAS | + +**Chunking strategy:** + +For large graphs (>10MB): + +```json +{ + "chunked_graph": { + "chunk_count": 5, + "chunks": [ + {"chunk_id": "chunk:001", "uri": "cas://graphs/chunks/001", "hash": "blake3:..."}, + {"chunk_id": "chunk:002", "uri": "cas://graphs/chunks/002", "hash": "blake3:..."} + ], + "assembly_order": ["chunk:001", "chunk:002", ...], + "assembled_hash": "blake3:..." + } +} +``` + +**Compression:** + +- Graph JSON: gzip before DSSE +- CAS storage: Raw JSON (indexed) +- Rekor payload: DSSE references CAS + +### BR9: API/CLI/UI Surfacing + +**API endpoints:** + +| Method | Path | Description | +|--------|------|-------------| +| `POST` | `/api/binary/graphs` | Submit binary graph | +| `GET` | `/api/binary/graphs/{hash}` | Get graph details | +| `GET` | `/api/binary/graphs/{hash}/edges` | List edges | +| `GET` | `/api/binary/symbols/{symbolId}` | Get symbol details | +| `POST` | `/api/binary/verify` | Verify graph attestation | + +**CLI commands:** + +```bash +# Submit binary graph +stella binary submit --graph ./richgraph.json --binary ./app + +# Get graph info +stella binary info --hash blake3:a1b2c3d4... + +# List symbols +stella binary symbols --hash blake3:... --stripped-only + +# Verify attestation +stella binary verify --graph ./richgraph.json --dsse ./richgraph.dsse +``` + +**UI components:** + +- Binary graph visualization with zoom/pan +- Symbol table with search/filter +- Edge explorer with confidence highlighting +- Attestation status badges +- Build variant selector + +### BR10: Binary Fixtures + +**Fixture location:** + +``` +tests/Binary/ + fixtures/ + elf-x86_64-with-debug/ + binary.elf + graph.json + expected-hashes.txt + elf-stripped/ + binary.elf + graph.json + expected-hashes.txt + pe-x64-with-pdb/ + binary.exe + graph.json + expected-hashes.txt + golden/ + elf-x86_64.golden.json + pe-x64.golden.json + +datasets/binary/ + schema/ + binary-graph.schema.json + binary-edge.schema.json + samples/ + openssl-1.1.1/ + libssl.so + graph.json + edges.ndjson +``` + +**Fixture requirements:** + +1. Each binary format has at least one fixture +2. Stripped and debug variants for each format +3. Expected hashes verified by CI +4. Golden outputs include DSSE envelopes +5. Fixtures reproducible from source (where legal) + +**Test categories:** + +1. **Hash stability:** Same binary produces same graph hash +2. **Build-ID extraction:** Correct build-ID parsing per format +3. **Symbol recovery:** DWARF/PDB parsing accuracy +4. **Stripped handling:** Code block hash computation +5. **Chunking:** Large graph assembly/disassembly +6. **DSSE signing:** Envelope creation and verification +7. **Rekor integration:** Submission and verification + +--- + +## 3. Implementation Status + +| Component | Location | Status | +|-----------|----------|--------| +| ELF parser | `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Native` | Implemented | +| PE parser | `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Native` | Implemented | +| DSSE predicates | `src/Signer/StellaOps.Signer/PredicateTypes.cs` | Implemented | +| CAS storage | `src/Scanner/__Libraries/StellaOps.Scanner.Reachability` | Partial | +| Rekor integration | `src/Attestor/StellaOps.Attestor` | Implemented | +| CLI commands | `src/Cli/StellaOps.Cli` | Planned | +| UI components | `src/UI/StellaOps.UI` | Planned | + +--- + +## 4. Related Documentation + +- [richgraph-v1 Contract](../contracts/richgraph-v1.md) - Graph schema specification +- [Function-Level Evidence](./function-level-evidence.md) - Evidence chain guide +- [Edge Explainability](./edge-explainability-schema.md) - Edge reason codes +- [Hybrid Attestation](./hybrid-attestation.md) - Graph and edge-bundle DSSE +- [Native Analyzer Tests](../../src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Native.Tests/Reachability/) - Test fixtures + +--- + +_Last updated: 2025-12-13. See Sprint 0401 BINARY-GAPS-401-066 for change history._ diff --git a/docs/reachability/corpus-plan.md b/docs/reachability/corpus-plan.md index 9841ee210..093a1f67b 100644 --- a/docs/reachability/corpus-plan.md +++ b/docs/reachability/corpus-plan.md @@ -1,45 +1,69 @@ # Reachability Corpus Plan (QA-CORPUS-401-031) Objective -- Build a multi-runtime reachability corpus (Go/.NET/Python/Rust) with EXPECT.yaml ground truths and captured traces. -- Make fixtures CI-consumable to validate reachability scoring and VEX proofs continuously. -- Add public mini-dataset cases (PHP/JavaScript/C#) from advisory 23-Nov-2025 for ingestion/bench reuse. +- Maintain deterministic, offline reachability fixtures that validate callgraph ingestion, reachability truth-path handling, and VEX proof workflows. +- Keep the corpus small but multi-runtime (Go/.NET/Python/Rust), and keep a public-friendly mini dataset (PHP/JavaScript/C#) for docs/demos without external repos. -Scope & deliverables -- Fixture layout: `tests/reachability/corpus///` - - `expect.yaml` — states (`reachable|conditional|unreachable`), score, evidence refs. - - `callgraph.*.json` — static graphs per language. - - `runtime/*.ndjson` — traces/probes when available. - - `sbom.*.json` — CycloneDX/SPDX slices. - - `vex.openvex.json` — expected VEX statement. -- CI integration: add corpus harness to `tests/reachability/StellaOps.Reachability.FixtureTests` to validate presence, schema, and determinism (hash manifest). -- Offline posture: all artifacts deterministic, no external downloads; hashes recorded in manifest. -- Public mini-dataset layout (PHP/JS/C#) to be mirrored under `tests/reachability/samples-public/`: -``` -vuln-reach-dataset/ - schema/ground-truth.schema.json - runners/run_all.sh - samples/ - php/php-001-phar-deserialize/... - js/js-002-yaml-unsafe-load/... - csharp/cs-001-binaryformatter-deserialize/... -``` -Each sample ships: minimal app, lockfile, SBOM (CycloneDX JSON), VEX, ground truth (EXPECT/JSON), repro script. +## Corpus Map -MVP slice (proposed) +### 1) Multi-runtime corpus (internal MVP) + +Path: `tests/reachability/corpus/` + +Per-case layout: `tests/reachability/corpus///` +- `callgraph.static.json` — static call graph sample (stub for MVP). +- `ground-truth.json` — expected reachability outcome and example path(s) (Reachbench truth schema v1; `schema_version=reachbench.reachgraph.truth/v1`). +- `vex.openvex.json` — expected VEX slice for the case. +- Optional (future): `runtime/*.ndjson`, `sbom.*.json` + +`tests/reachability/corpus/manifest.json` records deterministic SHA-256 hashes for required files in each case directory. + +### 2) Public mini dataset (PHP/JS/C#) + +Path: `tests/reachability/samples-public/` + +Layout: +- `schema/ground-truth.schema.json` — JSON schema for `ground-truth.json` (Reachbench truth schema v1). +- `manifest.json` — deterministic SHA-256 hashes for required files in each sample directory. +- `samples///` — per-sample artifacts: `callgraph.static.json`, `ground-truth.json`, `sbom.cdx.json`, `vex.openvex.json`, `repro.sh`. +- `runners/run_all.{sh,ps1}` — deterministic manifest regeneration. + +### 3) Reachbench fixture pack (expanded, dual variants) + +Path: `tests/reachability/fixtures/reachbench-2025-expanded/` + +Each case has two variants (reachable/unreachable) with per-variant `manifest.json` and `reachgraph.truth.json`. Fixture integrity is validated by `tests/reachability/StellaOps.Reachability.FixtureTests`. + +## Ground Truth Conventions + +- Corpus and public samples use the same truth schema (`reachbench.reachgraph.truth/v1`) but differ in file naming (`ground-truth.json` vs reachbench pack `reachgraph.truth.json`). +- Legacy corpus `expect.yaml` has been retired; prior `state/score` values are preserved under `legacy_expect` in `ground-truth.json`. +- Legacy `conditional` states are represented as `variant=unreachable` plus `legacy_expect.state=conditional` until the truth schema grows a dedicated conditional/contested variant. + +## Determinism & Runners + +Regenerate all reachability manifests (corpus + public samples + reachbench pack): +- `tests/reachability/runners/run_all.sh` +- `tests/reachability/runners/run_all.ps1` + +Individual scripts: +- `python tests/reachability/scripts/update_corpus_manifest.py` +- `python tests/reachability/samples-public/scripts/update_manifest.py` +- `python tests/reachability/fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py` + +## CI Gates + +- `tests/reachability/StellaOps.Reachability.FixtureTests` + - validates presence + hashes from manifests for corpus/public samples/reachbench fixtures + - enforces minimum language-bucket coverage (Go/.NET/Python/Rust + PHP/JS/C#) + +## MVP Slice (stub cases) - Go: `go-ssh-CVE-2020-9283-keyexchange` - .NET: `dotnet-kestrel-CVE-2023-44487-http2-rapid-reset` - Python: `python-django-CVE-2019-19844-sqli-like` - Rust: `rust-axum-header-parsing-TBD` -Work plan -1) Define shared manifest schema + hash manifest (NDJSON) under `tests/reachability/corpus/manifest.json`. -2) For each MVP case, add minimal static callgraph + EXPECT.yaml with score/state and evidence links. (DONE: stub versions committed) -3) Extend reachability fixture tests to cover corpus folders (presence, hashes, EXPECT.yaml schema). (DONE) -4) Wire CI job to run the extended tests in `tests/reachability/StellaOps.Reachability.FixtureTests`. (TODO) -5) Replace stubs with real callgraphs/traces and expand corpus after MVP passes CI. (TODO) +## Next Work (post-MVP) +- Wire a CI job to run `tests/reachability/StellaOps.Reachability.FixtureTests`. +- Replace stubs with real callgraphs/traces and expand the corpus once CI is stable. -Determinism rules -- Sort JSON keys; round scores to 2dp; UTC times only if needed. -- Stable ordering of files in manifests; hash with SHA-256. -- No network calls during test or generation. diff --git a/docs/reachability/edge-explainability-schema.md b/docs/reachability/edge-explainability-schema.md new file mode 100644 index 000000000..fc65b18ea --- /dev/null +++ b/docs/reachability/edge-explainability-schema.md @@ -0,0 +1,416 @@ +# Edge Explainability Schema + +_Last updated: 2025-12-13. Owner: Scanner Guild + Policy Guild._ + +This document defines the edge explainability schema addressing gaps EG1-EG10 from the November 2025 product findings. It specifies the canonical format for call edge evidence, reason codes, confidence rubrics, and propagation into explanation graphs and VEX. + +--- + +## 1. Overview + +Edge explainability provides detailed rationale for each call edge in the reachability graph. Every edge includes: + +- **Reason code:** Why this edge was detected (e.g., `bytecode-invoke`, `plt-stub`, `indirect-target`) +- **Confidence score:** Certainty of the edge's existence +- **Evidence sources:** Detectors and rules that contributed to edge discovery +- **Provenance:** Analyzer version, detection timestamp, and input artifacts + +--- + +## 2. Gap Resolutions + +### EG1: Reason Enum Governance + +**Standard reason codes:** + +| Code | Category | Description | Example | +|------|----------|-------------|---------| +| `bytecode-invoke` | Static | Bytecode invocation instruction | Java `invokevirtual`, .NET `call` | +| `bytecode-field` | Static | Field access leading to call | Static initializer | +| `import-symbol` | Static | Import table reference | ELF `.dynsym`, PE imports | +| `plt-stub` | Static | PLT/GOT indirection | `printf@plt` | +| `reloc-target` | Static | Relocation target | `.rela.dyn` entries | +| `indirect-target` | Heuristic | Indirect call target analysis | CFG-based | +| `init-array` | Static | Constructor/initializer array | `.init_array`, `DT_INIT` | +| `fini-array` | Static | Destructor/finalizer array | `.fini_array`, `DT_FINI` | +| `vtable-slot` | Heuristic | Virtual method dispatch | C++ vtable | +| `reflection-invoke` | Heuristic | Reflective method invocation | `Method.invoke()` | +| `runtime-observed` | Runtime | Runtime probe observation | JFR, eBPF | +| `user-annotated` | Manual | User-provided edge | Policy override | + +**Governance rules:** + +1. New reason codes require RFC + review by Scanner Guild +2. Deprecated codes remain valid for 2 major versions +3. Custom codes use `custom:` prefix (e.g., `custom:my-analyzer`) +4. Codes are case-insensitive, normalized to lowercase + +**Code registry:** + +```json +{ + "schema": "stellaops.edge.reason.registry@v1", + "version": "2025-12-13", + "reasons": [ + { + "code": "bytecode-invoke", + "category": "static", + "description": "Bytecode invocation instruction", + "languages": ["java", "dotnet"], + "confidence_range": [0.9, 1.0], + "deprecated": false + } + ] +} +``` + +### EG2: Canonical Edge Schema with Hash Rules + +**Edge schema:** + +```json +{ + "edge_id": "edge:sha256:{hex}", + "from": "sym:java:...", + "to": "sym:java:...", + "kind": "call", + "reason": "bytecode-invoke", + "confidence": 0.95, + "evidence": [ + { + "source": "detector:java-bytecode-analyzer", + "rule_id": "invoke-virtual", + "rule_version": "1.0.0", + "location": { + "file": "com/example/Foo.class", + "offset": 1234, + "instruction": "invokevirtual #42" + }, + "timestamp": "2025-12-13T10:00:00Z" + } + ], + "attributes": { + "virtual": true, + "polymorphic_targets": 3 + } +} +``` + +**Hash computation:** + +``` +edge_id = "edge:" + sha256( + canonical_json({ + "from": edge.from, + "to": edge.to, + "kind": edge.kind, + "reason": edge.reason + }) +) +``` + +**Canonicalization:** + +1. Use only `from`, `to`, `kind`, `reason` for hash (not confidence or evidence) +2. Sort JSON keys alphabetically +3. No whitespace, UTF-8 encoding +4. Hash is lowercase hex with `sha256:` prefix + +### EG3: Evidence Limits/Redaction + +**Evidence limits:** + +| Element | Default Limit | Configurable | +|---------|--------------|--------------| +| Evidence entries per edge | 10 | Yes | +| Location detail fields | 5 | Yes | +| Instruction preview length | 100 chars | Yes | +| File path depth | 10 segments | No | + +**Redaction rules:** + +| Category | Redaction | Example | +|----------|-----------|---------| +| File paths | Normalize | `/home/user/...` -> `{PROJECT}/...` | +| Bytecode offsets | Keep | Offsets are not PII | +| Instruction text | Truncate | First 100 chars | +| Source line content | Omit | Not included by default | + +**Truncation behavior:** + +```json +{ + "evidence_truncated": true, + "evidence_count": 15, + "evidence_shown": 10, + "full_evidence_uri": "cas://edges/evidence/sha256:..." +} +``` + +### EG4: Confidence Rubric + +**Confidence scale:** + +| Level | Range | Description | Typical Sources | +|-------|-------|-------------|-----------------| +| `certain` | 1.0 | Definite edge | Direct bytecode invoke | +| `high` | 0.85-0.99 | Very likely | Import table, PLT | +| `medium` | 0.5-0.84 | Probable | Indirect analysis, vtable | +| `low` | 0.2-0.49 | Possible | Heuristic carving | +| `unknown` | 0.0-0.19 | Speculative | User annotation, fallback | + +**Confidence computation:** + +``` +edge.confidence = base_confidence(reason) * evidence_boost(evidence_count) * target_resolution_factor +``` + +**Base confidence by reason:** + +| Reason | Base Confidence | +|--------|-----------------| +| `bytecode-invoke` | 0.98 | +| `import-symbol` | 0.95 | +| `plt-stub` | 0.92 | +| `reloc-target` | 0.90 | +| `init-array` | 0.95 | +| `vtable-slot` | 0.75 | +| `indirect-target` | 0.60 | +| `reflection-invoke` | 0.50 | +| `runtime-observed` | 0.99 | +| `user-annotated` | 0.80 | + +### EG5: Detector/Rule Provenance + +**Provenance schema:** + +```json +{ + "provenance": { + "analyzer": { + "name": "scanner.java", + "version": "1.2.0", + "digest": "sha256:..." + }, + "detector": { + "name": "java-bytecode-analyzer", + "version": "2.0.0", + "rule_set": "default" + }, + "rule": { + "id": "invoke-virtual", + "version": "1.0.0", + "description": "Detect invokevirtual bytecode instructions" + }, + "input_artifacts": [ + {"type": "jar", "digest": "sha256:...", "path": "lib/app.jar"} + ], + "detected_at": "2025-12-13T10:00:00Z" + } +} +``` + +**Provenance requirements:** + +1. All edges must include analyzer provenance +2. Detector/rule provenance required for non-runtime edges +3. Input artifact digests enable reproducibility +4. Detection timestamp uses UTC ISO-8601 + +### EG6: API/CLI Parity + +**API endpoints:** + +| Method | Path | Description | +|--------|------|-------------| +| `GET` | `/api/edges/{edgeId}` | Get edge details | +| `GET` | `/api/edges?graph_hash=...` | List edges for graph | +| `GET` | `/api/edges/{edgeId}/evidence` | Get full evidence | +| `POST` | `/api/edges/search` | Search edges by criteria | + +**CLI commands:** + +```bash +# List edges for a graph +stella edge list --graph blake3:a1b2c3d4... + +# Get edge details +stella edge show --id edge:sha256:... + +# Search edges +stella edge search --from "sym:java:..." --reason bytecode-invoke + +# Export edges +stella edge export --graph blake3:... --output ./edges.ndjson +``` + +**Output parity:** + +- API and CLI return identical JSON structure +- CLI supports `--json` for machine-readable output +- Both support filtering by reason, confidence, from/to + +### EG7: Deterministic Fixtures + +**Fixture location:** + +``` +tests/Edge/ + fixtures/ + bytecode-invoke.json + plt-stub.json + vtable-dispatch.json + init-array-constructor.json + runtime-observed.json + golden/ + bytecode-invoke.golden.json + graph-with-edges.golden.json + +datasets/edges/ + schema/ + edge.schema.json + reason-registry.json + samples/ + java-spring-boot/ + edges.ndjson + expected-hashes.txt +``` + +**Fixture requirements:** + +1. Each reason code has at least one fixture +2. Fixtures include expected `edge_id` hash +3. Golden outputs frozen after review +4. CI verifies hash stability + +### EG8: Propagation into Explanation Graphs/VEX + +**Explanation graph inclusion:** + +```json +{ + "explanation": { + "path": [ + { + "node": "sym:java:main...", + "outgoing_edge": { + "edge_id": "edge:sha256:...", + "to": "sym:java:handler...", + "reason": "bytecode-invoke", + "confidence": 0.98 + } + }, + { + "node": "sym:java:handler...", + "outgoing_edge": { + "edge_id": "edge:sha256:...", + "to": "sym:java:log4j...", + "reason": "bytecode-invoke", + "confidence": 0.95 + } + } + ], + "aggregate_path_confidence": 0.93 + } +} +``` + +**VEX evidence format:** + +```json +{ + "stellaops:reachability": { + "path_edges": [ + {"edge_id": "edge:sha256:...", "reason": "bytecode-invoke", "confidence": 0.98}, + {"edge_id": "edge:sha256:...", "reason": "bytecode-invoke", "confidence": 0.95} + ], + "weakest_edge": { + "edge_id": "edge:sha256:...", + "reason": "bytecode-invoke", + "confidence": 0.95 + }, + "aggregate_confidence": 0.93 + } +} +``` + +### EG9: Localization Guidance + +**Localizable elements:** + +| Element | Localization | Example | +|---------|--------------|---------| +| Reason code display | Message catalog | `bytecode-invoke` -> "Bytecode method call" | +| Confidence level | Message catalog | `high` -> "High confidence" | +| Evidence descriptions | Template | "Detected at offset {offset} in {file}" | +| Error messages | Message catalog | Standard error codes | + +**Message catalog structure:** + +```json +{ + "locale": "en-US", + "messages": { + "edge.reason.bytecode-invoke": "Bytecode method call", + "edge.reason.plt-stub": "PLT/GOT library call", + "edge.confidence.high": "High confidence ({0:P0})", + "edge.evidence.location": "Detected at offset {offset} in {file}" + } +} +``` + +**Supported locales:** + +- `en-US` (default) +- Additional locales via contribution + +### EG10: Backfill Plan + +**Backfill strategy:** + +1. **Phase 1:** Add reason codes to new edges (no backfill needed) +2. **Phase 2:** Run detector upgrade on graphs without reason codes +3. **Phase 3:** Mark old graphs as `requires_reanalysis` in metadata + +**Migration script:** + +```bash +stella edge backfill --graph blake3:... --dry-run + +# Output: +Graph: blake3:a1b2c3d4... +Edges without reason: 1234 +Edges to update: 1234 + +Dry run - no changes made. + +# Execute: +stella edge backfill --graph blake3:... --execute +``` + +**Backfill metadata:** + +```json +{ + "backfill": { + "status": "complete", + "original_analyzer_version": "1.0.0", + "backfill_analyzer_version": "1.2.0", + "backfilled_at": "2025-12-13T10:00:00Z", + "edges_updated": 1234 + } +} +``` + +--- + +## 3. Related Documentation + +- [richgraph-v1 Contract](../contracts/richgraph-v1.md) - Graph schema specification +- [Function-Level Evidence](./function-level-evidence.md) - Evidence chain guide +- [Explainability Schema](./explainability-schema.md) - Explanation format +- [Hybrid Attestation](./hybrid-attestation.md) - Edge bundle DSSE + +--- + +_Last updated: 2025-12-13. See Sprint 0401 EDGE-GAPS-401-065 for change history._ diff --git a/docs/reachability/explainability-schema.md b/docs/reachability/explainability-schema.md new file mode 100644 index 000000000..37c0543a2 --- /dev/null +++ b/docs/reachability/explainability-schema.md @@ -0,0 +1,454 @@ +# Explainability Schema + +_Last updated: 2025-12-13. Owner: Policy Guild + Docs Guild._ + +This document defines the explainability schema addressing gaps EX1-EX10 from the November 2025 product findings. It specifies the canonical format for vulnerability verdict explanations, DSSE signing policy, CAS storage rules, and export/replay formats. + +--- + +## 1. Overview + +Explainability provides auditable, machine-readable rationale for every vulnerability verdict. Each explanation includes: + +- **Decision chain:** Ordered list of rules/policies that contributed to the verdict +- **Evidence links:** References to graphs, runtime facts, VEX statements, and SBOM components +- **Confidence scores:** Per-rule and aggregate confidence values +- **Redaction metadata:** PII handling and data classification + +--- + +## 2. Gap Resolutions + +### EX1: Schema/Canonicalization + Hashes + +**Explanation schema:** + +```json +{ + "schema": "stellaops.explanation@v1", + "explanation_id": "explain:sha256:{hex}", + "finding_id": "P-7:S-42:pkg:maven/log4j@2.14.1:CVE-2021-44228", + "verdict": { + "status": "affected", + "severity": {"normalized": "Critical", "score": 10.0}, + "confidence": 0.92 + }, + "decision_chain": [ + { + "rule_id": "rule:reachability_gate", + "rule_version": "1.0.0", + "inputs": { + "reachability.state": "CR", + "reachability.confidence": 0.92 + }, + "output": {"allowed": true, "contribution": 0.4}, + "evidence_refs": ["cas://reachability/graphs/blake3:..."] + }, + { + "rule_id": "rule:severity_baseline", + "rule_version": "1.0.0", + "inputs": { + "cvss_base": 10.0, + "epss_percentile": 0.95 + }, + "output": {"severity": "Critical", "contribution": 0.6}, + "evidence_refs": ["cas://advisories/CVE-2021-44228.json"] + } + ], + "aggregate_confidence": 0.88, + "created_at": "2025-12-13T10:00:00Z", + "policy_version": "sha256:...", + "graph_revision_id": "rev:blake3:..." +} +``` + +**Canonicalization rules:** + +1. JSON keys sorted alphabetically at all levels +2. Arrays in `decision_chain` ordered by rule execution sequence +3. `evidence_refs` arrays sorted alphabetically +4. No whitespace, UTF-8 encoding +5. Hash computed over canonical JSON: `sha256(canonical_json)` + +### EX2: DSSE Predicate/Signing Policy + +**DSSE predicate type:** + +``` +stella.ops/explanation@v1 +``` + +**Signing policy:** + +| Element | Required | Signer | +|---------|----------|--------| +| Explanation body | Yes | Policy Engine key | +| Graph DSSE reference | Yes (if reachability cited) | Scanner key | +| VEX DSSE reference | Yes (if VEX cited) | Policy Engine key | + +**DSSE envelope structure:** + +```json +{ + "payloadType": "application/vnd.stellaops.explanation+json", + "payload": "", + "signatures": [ + { + "keyid": "policy-engine-signing-2025", + "sig": "base64:..." + } + ] +} +``` + +**Signing requirements:** + +- All explanations must be signed before CAS storage +- Signing key must be registered in Authority key store +- Key rotation triggers re-signing of active explanations (configurable) + +### EX3: CAS Storage Rules for Evidence + +**Storage layout:** + +``` +cas://explanations/ + {sha256}/ # Explanation body + {sha256}.dsse # DSSE envelope + by-finding/{finding_id}/ # Index by finding + by-policy/{policy_digest}/ # Index by policy version + by-graph/{graph_revision_id}/ # Index by graph revision +``` + +**Storage rules:** + +1. Explanations are immutable after signing +2. New verdicts create new explanation documents (no updates) +3. Previous explanations are retained per retention policy +4. Cross-references validated at write time (graphs, VEX must exist) + +**Deduplication:** + +- Identical canonical JSON produces identical hash +- CAS returns existing reference if content matches + +### EX4: Link to Decision/Policy and graph_revision_id + +**Required links:** + +```json +{ + "links": { + "policy_version": "sha256:7e1d...", + "policy_uri": "cas://policy/versions/sha256:7e1d...", + "graph_revision_id": "rev:blake3:a1b2...", + "graph_uri": "cas://reachability/revisions/blake3:a1b2...", + "sbom_digest": "sha256:def4...", + "sbom_uri": "cas://scanner-artifacts/sbom.cdx.json", + "vex_digest": "sha256:e5f6...", + "vex_uri": "cas://excititor/vex/openvex.json" + } +} +``` + +**Validation:** + +- All linked artifacts must exist at explanation creation time +- Links are verified during replay/audit +- Broken links cause replay verification failure + +### EX5: Export/Replay Bundle Format + +**Export bundle manifest:** + +```json +{ + "schema": "stellaops.explanation.bundle@v1", + "bundle_id": "bundle:explain:2025-12-13", + "created_at": "2025-12-13T10:00:00Z", + "explanations": [ + { + "explanation_id": "explain:sha256:...", + "finding_id": "...", + "explanation_uri": "explanations/sha256:....json", + "dsse_uri": "explanations/sha256:....dsse" + } + ], + "dependencies": { + "graphs": [ + {"revision_id": "rev:blake3:...", "uri": "graphs/blake3:....json"} + ], + "policies": [ + {"digest": "sha256:...", "uri": "policies/sha256:....json"} + ], + "vex_statements": [ + {"digest": "sha256:...", "uri": "vex/sha256:....json"} + ] + }, + "verification": { + "bundle_hash": "sha256:...", + "signature": "base64:...", + "signed_by": "policy-engine-signing-2025" + } +} +``` + +**Replay verification:** + +```bash +stella explain verify --bundle ./explanation-bundle.tgz + +# Output: +Bundle: bundle:explain:2025-12-13 +Explanations: 42 +Dependencies: 5 graphs, 2 policies, 12 VEX + +Verifying explanations... + Canonical hashes: 42/42 MATCH + DSSE signatures: 42/42 VALID + Dependency links: 42/42 RESOLVED + +Replay verification PASSED. +``` + +### EX6: PII/Redaction Rules + +**Redaction categories:** + +| Category | Redaction | Example | +|----------|-----------|---------| +| User identifiers | Hash | `user:alice` -> `user:sha256:a1b2...` | +| IP addresses | Mask | `192.168.1.100` -> `192.168.x.x` | +| File paths | Normalize | `/home/alice/code/...` -> `{HOME}/code/...` | +| Email addresses | Hash | `alice@example.com` -> `email:sha256:...` | +| API keys/tokens | Omit | `Authorization: Bearer xxx` -> `[REDACTED]` | + +**Redaction metadata:** + +```json +{ + "redaction": { + "applied": true, + "level": "standard", + "fields_redacted": ["actor.email", "evidence.file_path"], + "redaction_policy": "stellaops.redaction.standard@v1" + } +} +``` + +**Export modes:** + +- `--redacted` (default): Apply standard redaction +- `--full`: Include all data (requires `explain:export:full` scope) +- `--audit`: Include redaction audit trail + +### EX7: Size Budgets + +**Limits:** + +| Element | Default Limit | Configurable | +|---------|--------------|--------------| +| Explanation body | 256 KB | Yes | +| Decision chain entries | 100 | Yes | +| Evidence refs per rule | 20 | Yes | +| Total evidence refs | 200 | Yes | +| Path entries | 50 | No | + +**Truncation behavior:** + +When limits are exceeded: +1. Log warning with truncation details +2. Add `truncation` metadata to explanation +3. Store full evidence in separate CAS object +4. Include `full_evidence_uri` reference + +```json +{ + "truncation": { + "applied": true, + "elements_truncated": ["decision_chain", "evidence_refs"], + "full_evidence_uri": "cas://explanations/full/sha256:..." + } +} +``` + +### EX8: Versioning + +**Schema versioning:** + +- Schema version in `schema` field: `stellaops.explanation@v1` +- Breaking changes increment major version +- Minor changes (additive fields) use v1.x +- Backward compatibility maintained for 2 major versions + +**Migration support:** + +```bash +stella explain migrate --from v1 --to v2 --input ./explanations/ + +# Output: +Migrating 1000 explanations from v1 to v2... + Migrated: 998 + Skipped (already v2): 2 + +Migration complete. +``` + +**Version compatibility matrix:** + +| API Version | Schema v1 | Schema v2 | +|-------------|-----------|-----------| +| 1.0.x | Full | N/A | +| 1.1.x | Full | Full | +| 2.0.x | Read-only | Full | + +### EX9: Golden Fixtures/Tests + +**Test fixture location:** + +``` +tests/Explanation/ + fixtures/ + simple-affected.json + simple-not-affected.json + with-reachability-evidence.json + multi-rule-chain.json + truncated-evidence.json + redacted-pii.json + golden/ + simple-affected.golden.json + simple-affected.golden.dsse + +datasets/explanations/ + schema/ + explanation.schema.json + samples/ + log4j-affected/ + explanation.json + expected-hash.txt +``` + +**Test categories:** + +1. **Canonicalization tests:** Verify hash stability across JSON reordering +2. **DSSE signing tests:** Verify signature creation and verification +3. **Redaction tests:** Verify PII handling +4. **Truncation tests:** Verify size budget enforcement +5. **Replay tests:** Verify bundle export/import cycle +6. **Migration tests:** Verify version upgrade paths + +**CI integration:** + +```yaml +# .gitea/workflows/explanation-tests.yml +explanation-tests: + runs-on: ubuntu-latest + steps: + - name: Run explanation tests + run: dotnet test src/Policy/__Tests/StellaOps.Policy.Explanation.Tests + - name: Verify golden fixtures + run: scripts/verify-golden-fixtures.sh tests/Explanation/golden/ +``` + +### EX10: Determinism Guarantees + +**Determinism requirements:** + +1. Same inputs produce identical `explanation_id` hash +2. Decision chain ordering is stable (execution order) +3. Evidence refs sorted alphabetically +4. Timestamps use UTC ISO-8601 with millisecond precision +5. Floating-point values rounded to 6 decimal places + +**Verification:** + +```bash +# Run twice with same inputs, verify identical hashes +stella explain generate --finding "..." --output a.json +stella explain generate --finding "..." --output b.json +diff a.json b.json # Should be empty + +# Or use built-in verify +stella explain verify-determinism --finding "..." --iterations 3 +``` + +--- + +## 3. API Reference + +### 3.1 Generate Explanation + +```http +POST /api/policy/findings/{findingId}/explain +Authorization: Bearer +Content-Type: application/json + +{ + "mode": "full", + "include_evidence": true, + "redaction_level": "standard" +} +``` + +### 3.2 Get Explanation + +```http +GET /api/explanations/{explanationId} +Authorization: Bearer +Accept: application/json +``` + +### 3.3 Export Explanation Bundle + +```http +POST /api/explanations/export +Authorization: Bearer +Content-Type: application/json + +{ + "finding_ids": ["...", "..."], + "include_dependencies": true, + "redaction_level": "standard" +} +``` + +### 3.4 Verify Explanation + +```http +POST /api/explanations/{explanationId}/verify +Authorization: Bearer +``` + +--- + +## 4. CLI Reference + +```bash +# Generate explanation for a finding +stella explain generate --finding "P-7:S-42:pkg:maven/log4j@2.14.1:CVE-2021-44228" + +# Export explanation bundle +stella explain export --findings ./finding-ids.txt --output ./bundle.tgz + +# Verify explanation +stella explain verify --explanation ./explanation.json --dsse ./explanation.dsse + +# Verify bundle +stella explain verify --bundle ./bundle.tgz + +# Check determinism +stella explain verify-determinism --finding "..." --iterations 5 +``` + +--- + +## 5. Related Documentation + +- [Function-Level Evidence](./function-level-evidence.md) - Evidence chain guide +- [Graph Revision Schema](./graph-revision-schema.md) - Graph versioning +- [Policy API](../api/policy.md) - Policy Engine REST API +- [DSSE Predicates](../modules/attestor/architecture.md) - Signing specifications + +--- + +_Last updated: 2025-12-13. See Sprint 0401 EXPLAIN-GAPS-401-064 for change history._ diff --git a/docs/reachability/function-level-evidence.md b/docs/reachability/function-level-evidence.md index f9aabbe9c..b0c29c978 100644 --- a/docs/reachability/function-level-evidence.md +++ b/docs/reachability/function-level-evidence.md @@ -1,175 +1,535 @@ -# Function-Level Evidence Readiness (Nov 2025 Advisory) +# Function-Level Evidence Guide -_Last updated: 2025-11-12. Owner: Business Analysis Guild._ +_Last updated: 2025-12-13. Owner: Docs Guild._ -This memo captures the outstanding work required to make Stella Ops scanners emit stable, function-level evidence that matches the November 2025 advisory. It does **not** implement any code; instead it enumerates requirements, links them to sprint tasks, and spells out the schema/API updates that the next agent must land. +This guide documents the cross-module function-level evidence chain that enables provable reachability claims. It covers the schema, identifiers, API usage, CLI commands, and integration patterns for Scanner, Signals, Policy, and Replay. --- -## 1. Goal & Scope +## 1. Overview -**Goal.** Anchor every vulnerability finding to an immutable `{artifact_digest, code_id}` tuple plus optional symbol hints so replayers can prove reachability against stripped binaries. +StellaOps implements a **function-level evidence chain** that anchors every vulnerability finding to immutable identifiers (`code_id`, `symbol_id`, `graph_hash`) enabling: -**Scope.** Scanner analyzers, runtime ingestion, Signals scoring, Replay manifests, Policy/VEX emission, CLI/UI explainers, and documentation/runbooks needed to operationalise the advisory. +- **Provable reachability:** Deterministic call-path evidence from entry points to vulnerable functions. +- **Stripped binary support:** `code_id` + `code_block_hash` provides identity when symbols are absent. +- **Evidence replay:** Sealed artifacts with DSSE attestation allow offline verification. +- **Cross-module linking:** Scanner -> Signals -> Policy -> VEX -> UI/CLI evidence chain. -Out of scope: implementing disassemblers or symbol servers; those will be handled inside the module-specific backlog tasks referenced below. +### 1.1 Core Identifiers + +| Identifier | Format | Purpose | Example | +|------------|--------|---------|---------| +| `symbol_id` | `sym:{lang}:{base64url}` | Canonical function identity | `sym:java:R3JlZXRpbmc...` | +| `code_id` | `code:{lang}:{base64url}` | Identity for name-less code blocks | `code:binary:YWJjZGVm...` | +| `graph_hash` | `blake3:{hex}` | Content-addressable graph identity | `blake3:a1b2c3d4e5f6...` | +| `symbol_digest` | `sha256:{hex}` | Hash of symbol_id for edge linking | `sha256:e5f6a7b8c9d0...` | +| `build_id` | `gnu-build-id:{hex}` | ELF/PE debug identifier | `gnu-build-id:5f0c7c3c...` | + +### 1.2 Evidence Chain Flow + +``` +Scanner -> richgraph-v1 -> Signals -> Scoring -> Policy -> VEX -> UI/CLI + | | | | | | | + | | | | | | +-- stella graph explain + | | | | | +-- OpenVEX with call-path proofs + | | | | +-- Policy gates + reachability.state + | | | +-- Lattice state + confidence + riskScore + | | +-- Runtime facts + static paths + | +-- BLAKE3 graph_hash + DSSE attestation + +-- code_id, symbol_id, build_id per node +``` --- -## 2. Advisory Requirements vs. System Gaps +## 2. Schema Reference -| Requirement | Current gap | Task references | Notes | -|-------------|-------------|-----------------|-------| -| Immutable code identity (`code_id` = `{format, build_id, start, length}` + optional `code_block_hash`) | Callgraph nodes are opaque strings with no address metadata. | Sprint 401 `GRAPH-CAS-401-001`, `GAP-SCAN-001`, `GAP-SYM-007` | `code_id` should live alongside existing `SymbolID` helpers so analyzers can emit it without duplicating logic. | -| Symbol hints (demangled name, source, confidence) | No schema fields for symbol metadata; demangling is ad-hoc per analyzer. | `GAP-SYM-007` | Require deterministic casing + `symbol.source ∈ {DWARF,PDB,SYM,none}`. | -| Runtime facts mapped to code anchors | `/signals/runtime-facts` now accepts JSON and NDJSON (gzip) streams, stores symbol/code/process/container metadata. | Sprint 400 `ZASTAVA-REACH-201-001`, Sprint 401 `SIGNALS-RUNTIME-401-002`, `GAP-ZAS-002`, `GAP-SIG-003` | Provenance enrichment (process/socket/container) persisted; next step is exposing CAS URIs + context facts and emitting events for Policy/Replay. | -| Replay/DSSE coverage | Replay manifests don’t enforce hash/CAS registration for graphs/traces. | Sprint 400 `REPLAY-REACH-201-005`, Sprint 401 `REPLAY-401-004`, `GAP-REP-004` | Extend manifest v2 with analyzer versions + BLAKE3 digests; add DSSE predicate types. | -| Policy/VEX/UI explainability | Policy uses coarse `reachability:*` tags; UI/CLI cannot show call paths or evidence hashes. | Sprint 401 `POLICY-VEX-401-006`, `UI-CLI-401-007`, `GAP-POL-005`, `GAP-VEX-006`, `EXPERIENCE-GAP-401-012` | Evidence blocks must cite `code_id`, graph hash, runtime CAS URI, analyzer version. | -| Operator documentation & samples | No guide shows how to replay `{build_id,start,len}` across CLI/API. | Sprint 401 `QA-DOCS-401-008`, `GAP-DOC-008` | Produce samples under `samples/reachability/**` plus CLI walkthroughs. | -| Build-id propagation | Build-id not consistently captured or threaded into `SymbolID`/`code_id`; SBOM/runtime joins are brittle. | Sprint 401 `SCANNER-BUILDID-401-035` | Capture `.note.gnu.build-id`, include in code identity, expose in SBOM exports and runtime events. | -| Load-time constructors as roots | Graph roots omit `.preinit_array`/`.init_array`/`_init`, missing load-time edges. | Sprint 401 `SCANNER-INITROOT-401-036` | Add synthetic roots with `phase=load`; include `DT_NEEDED` deps’ constructors. | -| PURL-resolved edges | Call edges do not carry `purl` or `symbol_digest`, slowing SBOM joins. | Sprint 401 `GRAPH-PURL-401-034` | Annotate edges per `docs/reachability/purl-resolved-edges.md`; keep deterministic graph hash. | -| Unknowns handling | Unresolved symbols/edges disappear silently. | Sprint 0400 `SIGNALS-UNKNOWN-201-008` | Emit Unknowns records (see `docs/signals/unknowns-registry.md`) and feed `unknowns_pressure` into scoring. | -| Patch-oracle QA | No guard-rail tests proving binary analyzers see real patch deltas. | Sprint 401 `QA-PORACLE-401-037` | Add paired vuln/fixed fixtures and expectations; wire to CI using `docs/reachability/patch-oracles.md`. | +### 2.1 SymbolID Construction ---- +Per-language canonical tuple format (NUL-separated, then SHA-256 -> base64url): -## 3. Workstreams & Expectations +| Language | Tuple Components | Example | +|----------|------------------|---------| +| Java | `{package}\0{class}\0{method}\0{descriptor}` | `com.example\0Foo\0bar\0(Ljava/lang/String;)V` | +| .NET | `{assembly}\0{namespace}\0{type}\0{member_signature}` | `MyApp\0Controllers\0UserController\0GetById(int)` | +| Go | `{module}\0{package}\0{receiver}\0{func}` | `github.com/user/repo\0handler\0*Server\0Handle` | +| Node | `{pkg_or_path}\0{export_path}\0{kind}` | `lodash\0get\0function` | +| Binary | `{file_hash}\0{section}\0{addr}\0{name}\0{linkage}\0{code_block_hash?}` | `sha256:abc...\0.text\00x401000\0ssl3_read\0global\0` | +| Python | `{pkg_or_path}\0{module}\0{qualified_name}` | `requests\0api\0get` | +| Ruby | `{gem_or_path}\0{module}\0{method}` | `rails\0ActionController::Base\0render` | +| PHP | `{composer_pkg}\0{namespace}\0{qualified_name}` | `symfony/http-kernel\0Kernel\0handle` | -### 3.1 Scanner Symbolization (GAP-SCAN-001 / GAP-SYM-007) +### 2.2 CodeID Construction -* Define `SymbolID` helpers that glue together `{artifact_digest, file`, optional `section`, `addr`, `length`, `code_block_hash`}. -* Update analyzer contracts so every analyzer returns both `symbol_id` and `code_id`, with demangled names stored under the new `symbol` block. -* Persist the data into `richgraph-v1` payloads and attach CAS URIs via `StellaOps.Scanner.Reachability`. -* Deliver fixtures in `tests/reachability/StellaOps.ScannerSignals.IntegrationTests` that prove determinism (same hash when analyzer flags reorder). -* **Helper status (2025-12-02):** `SymbolId.ForBinaryAddressed` + `CodeId.ForBinarySegment` now encode `{file_hash, section, addr, name, linkage, length, code_block_hash}` with normalized hex addresses. Analyzers should start emitting these tuples instead of ad-hoc hashes. -* **Binary lifter (2025-12-03):** `BinaryReachabilityLifter` emits richgraph nodes for ELF/PE/Mach-O using file SHA-256 + section/address tuples, attaches `code_id` anchors, and turns imports/load commands into `import` edges. -* **Schema wiring (2025-12-12):** `reachability-union` + `richgraph-v1` serializers now emit `symbol {mangled,demangled,source,confidence}` and optional `code_block_hash` for stripped blocks; confidence is clamped to `[0,1]` and `source` normalized to uppercase (`DWARF|PDB|SYM|NONE`). +For stripped binaries or name-less code blocks: -### 3.2 Runtime + Signals (GAP-ZAS-002 / GAP-SIG-003) +``` +code:{lang}:{base64url_sha256(format + file_hash + addr + length + section + code_block_hash)} +``` -* Extend Zastava Observer NDJSON schema to emit: `symbol_id`, `code_id`, `hit_count`, `observed_at`, `loader_base`, `process.buildId`. -* Implement `/signals/runtime-facts` ingestion (gzip + NDJSON) with CAS-backed storage under `cas://reachability/runtime/{sha256}`. -* Update `ReachabilityScoringService` to lattice states and include runtime evidence references plus CAS URIs in `ReachabilityFactDocument.Metadata`. +Example for stripped ELF: +``` +code:binary:YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo +``` -### 3.3 Replay & Evidence (GAP-REP-004) +### 2.3 Graph Node Schema -* Enforce CAS registration + BLAKE3 hashing before manifest writes (graphs and traces). -* Teach `ReachabilityReplayWriter` to require analyzer name/version, graph kind, `code_id` coverage summary. -* Update `docs/replay/DETERMINISTIC_REPLAY.md` once schema v2 is finalized. - -### 3.4 Policy, VEX, CLI/UI (GAP-POL-005 / GAP-VEX-006) - -* Policy Engine: ingest new reachability facts, expose `reachability.state`, `max_path_conf`, and `evidence.graph_hash` via SPL + API. -* CLI/UI: add `stella graph explain` and explain drawer showing call path (`SymbolID` list), code anchors, runtime hits, DSSE references. -* Notify templates: include short evidence summary (first hop + truncated `code_id`). - -### 3.5 Documentation & Samples (GAP-DOC-008) - -* Publish schema diffs in `docs/data/evidence-schema.md` (new file) covering SBOM evidence nodes, runtime NDJSON, and API responses. -* Write CLI/API walkthroughs in `docs/09_API_CLI_REFERENCE.md` and `docs/api/policy.md` showing how to request reachability evidence and verify DSSE chains. -* Produce OpenVEX + replay samples under `samples/reachability/` showing `facts.type = "stella.reachability"` with `graph_hash` and `code_id` arrays. - -### 3.6 Native lifter & Reachability Store (SCANNER-NATIVE-401-015 / SIG-STORE-401-016) - -* Stand up `Scanner.Symbols.Native` + `Scanner.CallGraph.Native` libraries that: - * parse ELF (DWARF + `.symtab`/`.dynsym`), PE/COFF (CodeView/PDB), and stripped binaries via probabilistic carving; - * emit deterministic `FuncNode` + `CallEdge` records with demangled names, language hints, and `{confidence,evidence}` arrays; and - * attach analyzer + toolchain identifiers consumed by `richgraph-v1`. -* Introduce `Reachability.Store` collections in Mongo: - * `func_nodes` – keyed by `func:::` with `{binDigest,name,addr,size,lang,confidence,sym}`. - * `call_edges` – `{from,to,kind,confidence,evidence[]}` linking internal/external nodes. - * `cve_func_hits` – `{cve,purl,func_id,match_kind,confidence,source}` for advisory alignment. -* Build indexes (`binDigest+name`, `from→to`, `cve+func_id`) and expose repository interfaces so Scanner, Signals, and Policy can reuse the same canonical data without duplicating queries. - ---- - -## 4. Schema & API Touchpoints - -Authoritative field list lives in `docs/reachability/evidence-schema.md`; use it for DTOs and CAS writers. - -The next implementation pass must cover the following documents/files (create them if missing): - -1. `docs/data/evidence-schema.md` – authoritative schema for `{code_id, symbol, tool}` blocks. -2. `docs/runbooks/reachability-runtime.md` – operator steps for staging runtime ingestion bundles, retention, and troubleshooting. -3. `docs/runbooks/replay_ops.md` – add section detailing replay verification using the new graph/runtime CAS entries. - -API contracts to amend: - -- `POST /signals/callgraphs` response includes `graphHash` (sha256) for the normalized callgraph; richgraph-v1 uses BLAKE3 for graph CAS hashes. -- `POST /signals/runtime-facts` request body schema (NDJSON) with `symbol_id`, `code_id`, `hit_count`, `loader_base`. -- `GET /policy/findings` payload must surface `reachability.evidence[]` objects. - -### 4.1 Signals runtime ingestion snapshot (Nov 2025) - -- `/signals/runtime-facts` (JSON) and `/signals/runtime-facts/ndjson` (streaming, optional gzip) accept the following event fields: - - `symbolId` (required), `codeId`, `loaderBase`, `hitCount`, `processId`, `processName`, `socketAddress`, `containerId`, `evidenceUri`, `metadata`. - - Subject context (`scanId` / `imageDigest` / `component` / `version`) plus `callgraphId` is supplied either in the JSON body or as query params for the NDJSON endpoint. -- Signals dedupes events, merges metadata, and persists the aggregated `RuntimeFacts` onto `ReachabilityFactDocument`. These facts now feed reachability scoring (SIGNALS-24-004/005) as part of the runtime bonus lattice. -- Outstanding work: record CAS URIs for runtime traces, emit provenance events, and expose the enriched context to Policy/Replay consumers. - -### 4.2 Reachability store layout (SIG-STORE-401-016) - -All producers **must** persist native function evidence using the shared collections below (names are advisory; exact names live in Mongo options): +Each node in a richgraph-v1 document includes: ```json -// func_nodes { - "_id": "func:ELF:sha256:4012a0", - "binDigest": "sha256:deadbeef...", - "name": "ssl3_read_bytes", - "addr": "0x4012a0", - "size": 312, - "lang": "c", - "confidence": 0.92, - "symbol": { "mangled": "_Z15ssl3_read_bytes", "demangled": "ssl3_read_bytes", "source": "DWARF" }, - "sym": "present" -} - -// call_edges -{ - "from": "func:ELF:sha256:4012a0", - "to": "func:ELF:sha256:40f0ff", - "kind": "static", - "confidence": 0.88, - "evidence": ["reloc:.plt.got", "bb-target:0x40f0ff"] -} - -// cve_func_hits -{ - "cve": "CVE-2023-XXXX", - "purl": "pkg:generic/openssl@1.1.1u", - "func_id": "func:ELF:sha256:4012a0", - "match": "name+version", - "confidence": 0.77, - "source": "concelier:openssl-advisory" + "id": "sym:java:R3JlZXRpbmdTZXJ2aWNl...", + "symbol_id": "sym:java:R3JlZXRpbmdTZXJ2aWNl...", + "code_id": "code:java:...", + "lang": "java", + "kind": "method", + "display": "com.example.GreetingService.greet(String)", + "purl": "pkg:maven/com.example/greeting-service@1.0.0", + "build_id": "gnu-build-id:5f0c7c3c...", + "symbol_digest": "sha256:e5f6a7b8...", + "code_block_hash": "sha256:deadbeef...", + "symbol": { + "mangled": null, + "demangled": "com.example.GreetingService.greet(String)", + "source": "DWARF", + "confidence": 0.98 + }, + "evidence": ["import", "bytecode"], + "attributes": {} } ``` -Writers **must**: +### 2.4 Graph Edge Schema -1. Upsert `func_nodes` before emitting edges/hits to ensure `_id` lookups remain stable. -2. Serialize evidence arrays in deterministic order (`reloc`, `bb-target`, `import`, …) and normalise hex casing. -3. Attach analyzer fingerprints (`scanner.native@sha256:...`) so Replay/Policy can enforce provenance. +Edges carry callee `purl` and `symbol_digest` for SBOM correlation: + +```json +{ + "from": "sym:java:caller...", + "to": "sym:java:callee...", + "kind": "call", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1", + "symbol_digest": "sha256:f1e2d3c4...", + "confidence": 0.92, + "evidence": ["bytecode", "import"], + "candidates": [] +} +``` + +### 2.5 Evidence Block Schema + +Evidence blocks in Policy/VEX responses cite all relevant identifiers: + +```json +{ + "evidence": { + "graph_hash": "blake3:a1b2c3d4e5f6...", + "graph_cas_uri": "cas://reachability/graphs/a1b2c3d4e5f6...", + "dsse_uri": "cas://reachability/graphs/a1b2c3d4e5f6....dsse", + "path": [ + {"symbol_id": "sym:java:...", "display": "main()"}, + {"symbol_id": "sym:java:...", "display": "processRequest()"}, + {"symbol_id": "sym:java:...", "display": "log4j.error()"} + ], + "path_length": 3, + "confidence": 0.85, + "runtime_hits": ["probe:jfr:1234"], + "analyzer": { + "name": "scanner.java", + "version": "1.2.0", + "toolchain_digest": "sha256:..." + } + } +} +``` --- -## 5. Test & Fixture Expectations +## 3. API Usage -- **Reachbench fixtures**: update golden cases with `code_id` + `symbol` metadata. Ensure both reachable/unreachable variants still pass once graphs contain the richer IDs. -- **Signals unit tests**: add deterministic tests for lattice scoring + runtime evidence linking (`tests/reachability/StellaOps.Signals.Reachability.Tests`). -- **Replay tests**: extend `tests/reachability/StellaOps.Replay.Core.Tests` to assert manifest v2 serialization and hash enforcement. +### 3.1 Signals Callgraph Ingestion -All fixtures must remain deterministic: sort nodes/edges, normalise casing, and freeze timestamps in test data. +Submit a callgraph and receive a deterministic `graph_hash`: + +```http +POST /signals/callgraphs +Authorization: Bearer +Content-Type: application/json + +{ + "schema": "richgraph-v1", + "analyzer": {"name": "scanner.java", "version": "1.2.0"}, + "nodes": [...], + "edges": [...], + "roots": [...] +} +``` + +**Response:** + +```json +{ + "graphHash": "blake3:a1b2c3d4e5f6...", + "casUri": "cas://reachability/graphs/a1b2c3d4e5f6...", + "dsseUri": "cas://reachability/graphs/a1b2c3d4e5f6....dsse", + "nodeCount": 1247, + "edgeCount": 3891 +} +``` + +### 3.2 Signals Runtime Facts + +Submit runtime observations with `code_id` anchors: + +```http +POST /signals/runtime-facts/ndjson?scanId=scan-123&imageDigest=sha256:abc123 +Authorization: Bearer +Content-Type: application/x-ndjson +Content-Encoding: gzip + +{"symbolId":"sym:java:...","codeId":"code:java:...","hitCount":47,"loaderBase":"0x7f...","processId":1234,"observedAt":"2025-12-13T10:00:00Z"} +{"symbolId":"sym:java:...","codeId":"code:java:...","hitCount":12,"loaderBase":"0x7f...","processId":1234,"observedAt":"2025-12-13T10:00:01Z"} +``` + +**Response:** + +```json +{ + "accepted": 128, + "duplicates": 2, + "evidenceUri": "cas://reachability/runtime/sha256:xyz789..." +} +``` + +### 3.3 Fetch Reachability Facts + +Query reachability state for a subject: + +```http +GET /signals/facts/{subjectKey} +Authorization: Bearer +``` + +**Response:** + +```json +{ + "subjectKey": "scan:123:pkg:maven/log4j:2.14.1:CVE-2021-44228", + "metadata": { + "fact": { + "digest": "sha256:abc123...", + "version": 3 + } + }, + "states": [ + { + "symbol": "sym:java:...", + "latticeState": "CR", + "bucket": "runtime", + "confidence": 0.92, + "score": 0.78, + "path": ["sym:java:main...", "sym:java:process...", "sym:java:log4j..."], + "evidence": { + "static": {"graphHash": "blake3:...", "pathLength": 3, "confidence": 0.85}, + "runtime": {"probeId": "probe:jfr:1234", "hitCount": 47, "observedAt": "2025-12-13T10:00:00Z"} + } + } + ], + "score": 0.78, + "aggregateTier": "T2", + "riskScore": 0.65 +} +``` + +### 3.4 Policy Findings with Reachability Evidence + +```http +GET /api/policy/findings/{policyId}/{findingId}/explain?mode=verbose +Authorization: Bearer +``` + +**Response (excerpt):** + +```json +{ + "findingId": "P-7:S-42:pkg:maven/log4j@2.14.1:CVE-2021-44228", + "reachability": { + "state": "CR", + "confidence": 0.92, + "evidence": { + "graph_hash": "blake3:a1b2c3d4...", + "path": [ + {"symbol_id": "sym:java:...", "display": "main()"}, + {"symbol_id": "sym:java:...", "display": "Logger.error()"} + ], + "runtime_hits": 47, + "fact_digest": "sha256:abc123..." + } + }, + "steps": [ + {"rule": "reachability_gate", "state": "CR", "allowed": true}, + {"rule": "severity_baseline", "severity": {"normalized": "Critical", "score": 10.0}} + ] +} +``` --- -## 6. Handoff Checklist for the Next Agent +## 4. CLI Usage -1. Confirm sprint entries (`SPRINT_400` and `SPRINT_401`) remain in sync when moving `GAP-*` tasks to DOING/DONE. -2. Start with `GAP-SYM-007` (schema/helper implementation) because downstream work depends on the new `code_id` payload shape. -3. Once schema PR merges, coordinate with Signals + Policy guilds to align on CAS naming and DSSE predicates before wiring APIs. -4. Update the docs listed in §4 as each component lands; keep this file current with statuses and links to PRs/ADRs. -5. Before shipping, run the reachbench fixtures end-to-end and capture hashes for inclusion in replay docs. +### 4.1 Graph Explain Command -Keep this document updated as tasks change state; it is the authoritative hand-off note for the advisory. +View the call path and evidence for a finding: + +```bash +stella graph explain --finding "pkg:maven/log4j@2.14.1:CVE-2021-44228" --scan-id scan-123 + +# Output: +Finding: CVE-2021-44228 in pkg:maven/log4j@2.14.1 +Reachability: CONFIRMED_REACHABLE (CR) +Confidence: 0.92 +Graph Hash: blake3:a1b2c3d4e5f6... + +Call Path (3 hops): + 1. main() [sym:java:R3JlZXRpbmcuLi4=] + -> processRequest() [direct call] + 2. processRequest() [sym:java:cHJvY2Vzcy4uLg==] + -> Logger.error() [virtual call] + 3. Logger.error() [sym:java:bG9nNGouLi4=] + [VULNERABLE: CVE-2021-44228] + +Runtime Evidence: + - JFR probe hit: 47 times + - Last observed: 2025-12-13T10:00:00Z + +DSSE Attestation: cas://reachability/graphs/a1b2c3d4....dsse +``` + +### 4.2 Graph Export Command + +Export a reachability graph for offline analysis: + +```bash +stella graph export --scan-id scan-123 --output ./evidence-bundle/ + +# Creates: +# ./evidence-bundle/richgraph-v1.json # Canonical graph +# ./evidence-bundle/richgraph-v1.json.dsse # DSSE envelope +# ./evidence-bundle/meta.json # Metadata +# ./evidence-bundle/runtime-facts.ndjson # Runtime observations +``` + +### 4.3 Graph Verify Command + +Verify a graph's DSSE signature and Rekor inclusion: + +```bash +stella graph verify --graph ./evidence-bundle/richgraph-v1.json \ + --dsse ./evidence-bundle/richgraph-v1.json.dsse \ + --rekor-log + +# Output: +Graph Hash: blake3:a1b2c3d4e5f6... +DSSE Signature: VALID (key: scanner-signing-2025) +Rekor Entry: 12345678 (verified) +Timestamp: 2025-12-13T09:30:00Z +``` + +--- + +## 5. OpenVEX Integration + +### 5.1 OpenVEX with Reachability Evidence + +When Policy emits VEX decisions, reachability evidence is included: + +```json +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://stellaops.example/vex/2025-12-13/001", + "author": "StellaOps Policy Engine", + "timestamp": "2025-12-13T10:00:00Z", + "version": 1, + "statements": [ + { + "vulnerability": {"@id": "CVE-2021-44228"}, + "products": [{"@id": "pkg:oci/myapp@sha256:abc123..."}], + "status": "affected", + "justification": "vulnerable_code_in_container", + "impact_statement": "Vulnerable Log4j method reachable from main entry point.", + "action_statement": "Upgrade to log4j 2.17.1 or later.", + "stellaops:reachability": { + "state": "CR", + "confidence": 0.92, + "graph_hash": "blake3:a1b2c3d4e5f6...", + "path_length": 3, + "evidence_uri": "cas://reachability/graphs/a1b2c3d4..." + } + } + ] +} +``` + +### 5.2 VEX "not_affected" with Unreachability Evidence + +When code is provably unreachable: + +```json +{ + "statements": [ + { + "vulnerability": {"@id": "CVE-2023-XXXXX"}, + "products": [{"@id": "pkg:oci/myapp@sha256:abc123..."}], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "Vulnerable function not reachable from any entry point.", + "stellaops:reachability": { + "state": "CU", + "confidence": 0.88, + "graph_hash": "blake3:d4e5f6a7b8c9...", + "evidence_uri": "cas://reachability/graphs/d4e5f6a7b8c9...", + "runtime_observation_window": "72h", + "runtime_hits": 0 + } + } + ] +} +``` + +--- + +## 6. Replay Manifest v2 + +### 6.1 Manifest Structure + +Replay manifests now enforce BLAKE3 hashing and CAS registration: + +```json +{ + "schema": "stellaops.replay.manifest@v2", + "subject": "scan:123", + "generatedAt": "2025-12-13T10:00:00Z", + "hashAlg": "blake3", + "artifacts": [ + { + "kind": "richgraph", + "uri": "cas://reachability/graphs/blake3:a1b2c3d4e5f6...", + "hash": "blake3:a1b2c3d4e5f6...", + "dsseUri": "cas://reachability/graphs/blake3:a1b2c3d4e5f6....dsse" + }, + { + "kind": "runtime-facts", + "uri": "cas://reachability/runtime/sha256:xyz789...", + "hash": "sha256:xyz789..." + }, + { + "kind": "sbom", + "uri": "cas://scanner-artifacts/sbom.cdx.json", + "hash": "sha256:def456..." + } + ], + "analyzer": { + "name": "scanner.java", + "version": "1.2.0", + "toolchain_digest": "sha256:..." + }, + "code_id_coverage": { + "total_symbols": 1247, + "with_code_id": 1189, + "coverage_pct": 95.3 + } +} +``` + +### 6.2 Determinism Verification + +Replay a manifest to verify determinism: + +```bash +stella replay verify --manifest ./manifest.json --sealed + +# Output: +Manifest: stellaops.replay.manifest@v2 +Subject: scan:123 +Artifacts: 3 + +Verifying richgraph... + Computed: blake3:a1b2c3d4e5f6... + Expected: blake3:a1b2c3d4e5f6... + Status: MATCH + +Verifying runtime-facts... + Computed: sha256:xyz789... + Expected: sha256:xyz789... + Status: MATCH + +Verifying sbom... + Computed: sha256:def456... + Expected: sha256:def456... + Status: MATCH + +All artifacts verified. Determinism check PASSED. +``` + +--- + +## 7. Module Integration Guide + +### 7.1 Scanner -> Signals + +Scanner emits richgraph-v1 with `code_id` and `symbol_id`: + +1. Scanner analyzes container/artifact +2. Callgraph generators emit nodes with `symbol_id`, `code_id`, `build_id` +3. RichGraphWriter canonicalizes (sorted arrays/keys) and computes `graph_hash` (BLAKE3) +4. DSSE signer wraps canonical JSON +5. CAS store persists body + envelope +6. Signals ingestion API receives URI reference + +### 7.2 Signals -> Policy + +Signals provides reachability facts to Policy: + +1. Policy queries `/signals/facts/{subjectKey}` +2. Response includes `metadata.fact.digest`, `states[]`, `score` +3. Policy gates check `latticeState` (U, SR, SU, RO, RU, CR, CU, X) +4. Evidence blocks in findings reference `graph_hash`, `path[]`, `runtime_hits[]` + +### 7.3 Policy -> VEX/UI + +Policy emits OpenVEX with evidence: + +1. VexDecisionEmitter serializes OpenVEX with `stellaops:reachability` extension +2. UI explain drawer fetches evidence via `/api/policy/findings/{id}/explain` +3. CLI `stella graph explain` renders call path and attestation refs + +--- + +## 8. CAS Layout Reference + +``` +cas://reachability/ + graphs/ + {blake3}/ # Graph body (canonical JSON) + {blake3}.dsse # DSSE envelope + edges/ + {graph_hash}/{bundle_id} # Edge bundle body (optional) + {graph_hash}/{bundle_id}.dsse + runtime/ + {sha256}/ # Runtime facts NDJSON +``` + +--- + +## 9. Related Documentation + +- [Reachability Lattice Model](./lattice.md) - State definitions and join rules +- [richgraph-v1 Contract](../contracts/richgraph-v1.md) - Schema specification +- [Evidence Schema](./evidence-schema.md) - Detailed field definitions +- [Signals API Contract](../api/signals/reachability-contract.md) - API reference +- [Policy Gates](./policy-gate.md) - Gate configuration +- [Hybrid Attestation](./hybrid-attestation.md) - Graph and edge-bundle DSSE +- [Ground Truth Schema](./ground-truth-schema.md) - Test fixture format + +--- + +_Last updated: 2025-12-13. See Sprint 0401 GAP-DOC-008 for change history._ diff --git a/docs/reachability/graph-revision-schema.md b/docs/reachability/graph-revision-schema.md new file mode 100644 index 000000000..c1ee507fc --- /dev/null +++ b/docs/reachability/graph-revision-schema.md @@ -0,0 +1,377 @@ +# Graph Revision Schema + +_Last updated: 2025-12-13. Owner: Platform Guild._ + +This document defines the graph revision schema addressing gaps GR1-GR10 from the November 2025 product findings. It specifies manifest structure, hash algorithms, storage layout, lineage tracking, and governance rules for deterministic, auditable reachability graphs. + +--- + +## 1. Overview + +Graph revisions provide content-addressable, append-only versioning for `richgraph-v1` documents. Every graph mutation produces a new immutable revision with: + +- **Deterministic hash:** BLAKE3-256 of canonical JSON +- **Lineage metadata:** Parent revision + diff summary +- **Cross-artifact digests:** Links to SBOM, VEX, policy, and tool versions +- **Audit trail:** Timestamp, actor, tenant, and operation type + +--- + +## 2. Gap Resolutions + +### GR1: Manifest Schema + Canonical Hash Rules + +**Manifest schema:** + +```json +{ + "schema": "stellaops.graph.revision@v1", + "revision_id": "rev:blake3:a1b2c3d4e5f6...", + "graph_hash": "blake3:a1b2c3d4e5f6...", + "parent_revision_id": "rev:blake3:9f8e7d6c5b4a...", + "created_at": "2025-12-13T10:00:00Z", + "created_by": "service:scanner", + "tenant_id": "tenant:acme", + "shard_id": "shard:01", + "operation": "create", + "lineage": { + "depth": 3, + "root_revision_id": "rev:blake3:1a2b3c4d5e6f..." + }, + "cross_artifacts": { + "sbom_digest": "sha256:...", + "vex_digest": "sha256:...", + "policy_digest": "sha256:...", + "analyzer_digest": "sha256:..." + }, + "diff_summary": { + "nodes_added": 12, + "nodes_removed": 3, + "edges_added": 24, + "edges_removed": 8, + "roots_changed": false + } +} +``` + +**Canonical hash rules:** + +1. JSON keys sorted alphabetically at all nesting levels +2. No whitespace/indentation (compact JSON) +3. UTF-8 encoding, no BOM +4. Arrays sorted by deterministic key (nodes by `id`, edges by `from,to,kind`) +5. Null/empty values omitted +6. Numeric values without trailing zeros + +### GR2: Mandated BLAKE3-256 Encoding + +All graph-level hashes use BLAKE3-256 with the following format: + +``` +blake3:{64_hex_chars} +``` + +Example: +``` +blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 +``` + +**Rationale:** +- BLAKE3 is 3x+ faster than SHA-256 on modern CPUs +- Parallelizable for large graphs (>100K nodes) +- Cryptographically secure (256-bit security) +- Algorithm prefix enables future migration + +### GR3: Append-Only Storage + +Graph revisions are immutable. Operations: + +| Operation | Creates New Revision | Modifies Existing | +|-----------|---------------------|-------------------| +| `create` | Yes | No | +| `update` | Yes | No | +| `merge` | Yes | No | +| `tombstone` | Yes | No | +| `read` | No | No | + +**Storage layout:** + +``` +cas://reachability/ + revisions/ + {blake3}/ # Revision manifest + {blake3}.graph # Graph body + {blake3}.dsse # DSSE envelope + indices/ + by-tenant/{tenant_id}/ # Tenant index + by-sbom/{sbom_digest}/ # SBOM correlation + by-root/{root_revision_id}/ # Lineage tree +``` + +### GR4: Lineage/Diff Metadata + +Every revision tracks its lineage: + +```json +{ + "lineage": { + "depth": 5, + "root_revision_id": "rev:blake3:...", + "parent_revision_id": "rev:blake3:...", + "merge_parents": [] + }, + "diff_summary": { + "nodes_added": 12, + "nodes_removed": 3, + "nodes_modified": 0, + "edges_added": 24, + "edges_removed": 8, + "edges_modified": 0, + "roots_added": 0, + "roots_removed": 0 + }, + "diff_detail_uri": "cas://reachability/diffs/{parent_hash}_{child_hash}.ndjson" +} +``` + +**Diff detail format (NDJSON):** + +```ndjson +{"op":"add","path":"nodes","value":{"id":"sym:java:...","display":"..."}} +{"op":"remove","path":"edges","from":"sym:java:a","to":"sym:java:b"} +``` + +### GR5: Cross-Artifact Digests (SBOM/VEX/Policy/Tool) + +Every revision links to related artifacts: + +```json +{ + "cross_artifacts": { + "sbom_digest": "sha256:...", + "sbom_uri": "cas://scanner-artifacts/sbom.cdx.json", + "sbom_format": "cyclonedx-1.6", + "vex_digest": "sha256:...", + "vex_uri": "cas://excititor/vex/openvex.json", + "policy_digest": "sha256:...", + "policy_version": "P-7:v4", + "analyzer_digest": "sha256:...", + "analyzer_name": "scanner.java", + "analyzer_version": "1.2.0" + } +} +``` + +### GR6: UI/CLI Surfacing of Full/Short IDs + +**Full ID format:** +``` +rev:blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 +``` + +**Short ID format (for display):** +``` +rev:a1b2c3d4 +``` + +**CLI commands:** + +```bash +# List revisions +stella graph revisions --scan-id scan-123 + +# Show full ID +stella graph revisions --scan-id scan-123 --full + +# Output: +REVISION CREATED NODES EDGES PARENT +rev:a1b2c3d4 2025-12-13T10:00:00 1247 3891 rev:9f8e7d6c +rev:9f8e7d6c 2025-12-12T15:30:00 1235 3867 rev:1a2b3c4d +``` + +**UI display:** + +- Revision chips show short ID with copy-to-clipboard for full ID +- Hover tooltip shows full ID and creation timestamp +- Lineage tree visualization available in "Revision History" drawer + +### GR7: Shard/Tenant Context + +Every revision includes partition context: + +```json +{ + "tenant_id": "tenant:acme", + "shard_id": "shard:01", + "namespace": "prod", + "workspace_id": "ws:default" +} +``` + +**Tenant isolation:** + +- Revisions are tenant-scoped; cross-tenant access requires explicit grants +- Shard ID enables horizontal scaling and data locality +- Namespace supports multi-environment deployments + +### GR8: Pin/Audit Governance + +**Pinned revisions:** + +Revisions can be pinned to prevent automatic retention cleanup: + +```json +{ + "pinned": true, + "pinned_at": "2025-12-13T10:00:00Z", + "pinned_by": "user:alice", + "pin_reason": "Audit retention for CVE-2021-44228 investigation", + "pin_expires_at": "2026-12-13T10:00:00Z" +} +``` + +**Audit events:** + +All revision operations emit audit events: + +```json +{ + "event_type": "graph.revision.created", + "revision_id": "rev:blake3:...", + "actor": "service:scanner", + "tenant_id": "tenant:acme", + "timestamp": "2025-12-13T10:00:00Z", + "metadata": { + "operation": "create", + "parent_revision_id": "rev:blake3:...", + "graph_hash": "blake3:..." + } +} +``` + +### GR9: Retention/Tombstones + +**Retention policy:** + +| Category | Default Retention | Configurable | +|----------|-------------------|--------------| +| Latest revision | Forever | No | +| Intermediate revisions | 90 days | Yes | +| Tombstoned revisions | 30 days | Yes | +| Pinned revisions | Until unpin + 7 days | No | + +**Tombstone format:** + +```json +{ + "schema": "stellaops.graph.revision@v1", + "revision_id": "rev:blake3:...", + "tombstone": true, + "tombstoned_at": "2025-12-13T10:00:00Z", + "tombstoned_by": "service:retention-worker", + "tombstone_reason": "retention_policy", + "successor_revision_id": "rev:blake3:..." +} +``` + +### GR10: Inclusion in Offline Kits + +Offline kits include graph revisions for air-gapped deployments: + +**Offline bundle manifest:** + +```json +{ + "schema": "stellaops.offline.bundle@v1", + "bundle_id": "bundle:2025-12-13", + "graph_revisions": [ + { + "revision_id": "rev:blake3:...", + "graph_hash": "blake3:...", + "included_artifacts": ["graph", "dsse", "diff"] + } + ], + "rekor_checkpoints": [ + { + "log_id": "rekor.sigstore.dev", + "checkpoint": "...", + "verified_at": "2025-12-13T10:00:00Z" + } + ], + "signature": { + "algorithm": "ecdsa-p256", + "value": "base64:...", + "public_key_id": "key:offline-signing-2025" + } +} +``` + +**Import verification:** + +```bash +stella offline import --bundle ./offline-bundle.tgz --verify + +# Output: +Bundle: bundle:2025-12-13 +Graph Revisions: 5 +Rekor Checkpoints: 2 + +Verifying signatures... + Bundle signature: VALID + DSSE envelopes: 5/5 VALID + Rekor checkpoints: 2/2 VERIFIED + +Import complete. +``` + +--- + +## 3. API Reference + +### 3.1 Create Revision + +```http +POST /api/graph/revisions +Authorization: Bearer +Content-Type: application/json + +{ + "graph": { ... richgraph-v1 ... }, + "parent_revision_id": "rev:blake3:...", + "cross_artifacts": { ... } +} +``` + +### 3.2 Get Revision + +```http +GET /api/graph/revisions/{revision_id} +Authorization: Bearer +``` + +### 3.3 List Revisions + +```http +GET /api/graph/revisions?tenant_id=acme&sbom_digest=sha256:...&limit=20 +Authorization: Bearer +``` + +### 3.4 Diff Revisions + +```http +GET /api/graph/revisions/diff?from={rev_a}&to={rev_b} +Authorization: Bearer +``` + +--- + +## 4. Related Documentation + +- [richgraph-v1 Contract](../contracts/richgraph-v1.md) - Graph schema specification +- [Function-Level Evidence](./function-level-evidence.md) - Evidence chain guide +- [CAS Infrastructure](../contracts/cas-infrastructure.md) - Content-addressable storage +- [Offline Kit](../24_OFFLINE_KIT.md) - Air-gap deployment + +--- + +_Last updated: 2025-12-13. See Sprint 0401 GRAPHREV-GAPS-401-063 for change history._ diff --git a/docs/reachability/hybrid-attestation.md b/docs/reachability/hybrid-attestation.md index b154695f9..c7741c5b7 100644 --- a/docs/reachability/hybrid-attestation.md +++ b/docs/reachability/hybrid-attestation.md @@ -84,7 +84,93 @@ Stella Ops provides **true hybrid reachability** by combining: **Evidence linking:** Each edge in the graph or bundle includes `evidenceRefs` pointing to the underlying proof artifacts (static analysis artifacts, runtime traces), enabling **evidence-linked VEX decisions**. -## 8. Open decisions (tracked in Sprint 0401 tasks 53–56) -- Rekor publish defaults per deployment tier (regulated vs standard). -- CLI UX for selective bundle verification. -- Bench coverage for edge-bundle verification time/size. +## 8. Decisions (Frozen 2025-12-13) + +### 8.1 DSSE/Rekor Budget by Deployment Tier + +| Tier | Graph DSSE | Edge-Bundle DSSE | Rekor Publish | Max Bundles/Graph | +|------|------------|------------------|---------------|-------------------| +| **Regulated** (SOC2, FedRAMP, PCI) | Required | Required for runtime/contested | Required | 10 | +| **Standard** | Required | Optional (criteria-based) | Graph only | 5 | +| **Air-gapped** | Required | Optional | Offline checkpoint | 5 | +| **Dev/Test** | Optional | Optional | Disabled | Unlimited | + +**Budget enforcement:** +- Graph DSSE: Always submit digest to Rekor (or offline checkpoint for air-gapped) +- Edge-bundle DSSE: Submit to Rekor only when `bundle_reason` is `disputed`, `runtime-hit`, or `security-critical` +- Cap enforced by `reachability.edgeBundles.maxRekorPublishes` config (per tier defaults above) + +### 8.2 Signing Layout and CAS Paths + +``` +cas://reachability/ + graphs/ + {blake3}/ # richgraph-v1 body (JSON) + {blake3}.dsse # Graph DSSE envelope + {blake3}.rekor # Rekor inclusion proof (optional) + edges/ + {graph_hash}/ + {bundle_id}.json # Edge bundle body + {bundle_id}.dsse # Edge bundle DSSE envelope + {bundle_id}.rekor # Rekor inclusion proof (if published) + revisions/ + {revision_id}/ # Revision manifest + lineage +``` + +**Signing workflow:** +1. Canonicalize richgraph-v1 JSON (sorted keys, arrays by deterministic key) +2. Compute BLAKE3-256 hash -> `graph_hash` +3. Create DSSE envelope with `stella.ops/graph@v1` predicate +4. Submit digest to Rekor (online) or cache checkpoint (offline) +5. Store graph body + envelope + proof in CAS + +### 8.3 CLI UX for Selective Bundle Verification + +```bash +# Verify graph DSSE only (default) +stella graph verify --hash blake3:a1b2c3d4... + +# Verify graph + all edge bundles +stella graph verify --hash blake3:a1b2c3d4... --include-bundles + +# Verify specific edge bundle +stella graph verify --hash blake3:a1b2c3d4... --bundle bundle:001 + +# Offline verification with local CAS +stella graph verify --hash blake3:a1b2c3d4... --cas-root ./offline-cas/ + +# Verify Rekor inclusion +stella graph verify --hash blake3:a1b2c3d4... --rekor-proof + +# Output formats +stella graph verify --hash blake3:a1b2c3d4... --format json|table|summary +``` + +### 8.4 Golden Fixture Plan + +**Fixture location:** `tests/Reachability/Hybrid/` + +**Required fixtures:** +| Fixture | Description | Expected Verification Time | +|---------|-------------|---------------------------| +| `graph-only.golden.json` | Minimal richgraph-v1 with DSSE | < 100ms | +| `graph-with-runtime.golden.json` | Graph + 1 runtime edge bundle | < 200ms | +| `graph-with-contested.golden.json` | Graph + 1 contested/revoked edge bundle | < 200ms | +| `large-graph.golden.json` | 10K nodes, 50K edges, 5 bundles | < 2s | +| `offline-bundle.golden.tgz` | Complete offline replay pack | < 5s | + +**CI integration:** +- `.gitea/workflows/hybrid-attestation.yml` runs verification fixtures +- Size gate: Graph body < 10MB, individual bundle < 1MB +- Time gate: Full verification < 5s for standard tier + +### 8.5 Implementation Status + +| Component | Status | Notes | +|-----------|--------|-------| +| Graph DSSE predicate | Done | `stella.ops/graph@v1` in PredicateTypes.cs | +| Edge-bundle DSSE predicate | Planned | `stella.ops/edgeBundle@v1` | +| CAS layout | Done | Per section 8.2 | +| CLI verify command | Planned | Per section 8.3 | +| Golden fixtures | Planned | Per section 8.4 | +| Rekor integration | Done | Via Attestor module | diff --git a/docs/replay/replay-manifest-v2-acceptance.md b/docs/replay/replay-manifest-v2-acceptance.md new file mode 100644 index 000000000..2f5897b91 --- /dev/null +++ b/docs/replay/replay-manifest-v2-acceptance.md @@ -0,0 +1,311 @@ +# Replay Manifest v2 Acceptance Contract + +_Last updated: 2025-12-13. Owner: BE-Base Platform Guild._ + +This document defines the acceptance criteria and test vectors for replay manifest v2, enabling Task 19 (GAP-REP-004) to proceed with implementation. + +--- + +## 1. Overview + +Replay manifest v2 introduces: + +- **BLAKE3 graph hashes:** Primary hash algorithm for reachability graphs +- **Sorted CAS entries:** Deterministic ordering of all CAS references +- **hashAlg fields:** Explicit algorithm declarations for forward compatibility +- **code_id coverage:** Coverage metrics for stripped binary handling + +--- + +## 2. Schema Changes (v1 → v2) + +### 2.1 Version Field + +```json +{ + "schemaVersion": "2.0", + ... +} +``` + +### 2.2 Hash Algorithm Declaration + +All hash fields now include explicit algorithm: + +```json +{ + "reachability": { + "graphs": [ + { + "hash": "blake3:a1b2c3d4e5f6...", + "hashAlg": "blake3-256", + "casUri": "cas://reachability/graphs/blake3:a1b2c3d4..." + } + ], + "runtimeTraces": [ + { + "hash": "sha256:feedface...", + "hashAlg": "sha256", + "casUri": "cas://reachability/runtime/sha256:feedface..." + } + ] + } +} +``` + +### 2.3 Sorted CAS Entries + +All arrays must be sorted by deterministic key: + +| Array | Sort Key | +|-------|----------| +| `reachability.graphs[]` | `casUri` (lexicographic) | +| `reachability.runtimeTraces[]` | `casUri` (lexicographic) | +| `inputs.feeds[]` | `name` (lexicographic) | +| `inputs.tools[]` | `name` (lexicographic) | + +### 2.4 Code ID Coverage + +New field for stripped binary support: + +```json +{ + "reachability": { + "code_id_coverage": { + "total_nodes": 1247, + "nodes_with_symbol_id": 1189, + "nodes_with_code_id": 58, + "coverage_percent": 100.0 + } + } +} +``` + +--- + +## 3. CAS Registration Gates + +### 3.1 Required Registration + +All referenced artifacts must be registered in CAS before manifest finalization: + +| Artifact Type | CAS Path Pattern | Required | +|---------------|------------------|----------| +| Graph body | `cas://reachability/graphs/{hash}` | Yes | +| Graph DSSE | `cas://reachability/graphs/{hash}.dsse` | Yes | +| Runtime trace | `cas://reachability/runtime/{hash}` | Conditional | +| Edge bundle | `cas://reachability/edges/{graph_hash}/{bundle_id}` | Conditional | + +### 3.2 Registration Validation + +Before signing a replay manifest: + +1. Verify all `casUri` references resolve to existing CAS objects +2. Verify hash matches CAS content +3. Verify DSSE envelope exists for all graph references +4. Fail manifest creation if any reference is missing + +### 3.3 Validation API + +```csharp +public interface ICasValidator +{ + Task ValidateAsync(string casUri, string expectedHash); + Task ValidateBatchAsync(IEnumerable refs); +} + +public record CasValidationResult( + bool IsValid, + string? ActualHash, + string? Error +); +``` + +--- + +## 4. Acceptance Test Vectors + +### 4.1 Minimal Valid Manifest v2 + +```json +{ + "schemaVersion": "2.0", + "scan": { + "id": "scan-test-001", + "time": "2025-12-13T10:00:00Z", + "mode": "record", + "scannerVersion": "10.2.0" + }, + "subject": { + "ociDigest": "sha256:abc123..." + }, + "inputs": { + "feeds": [], + "tools": [] + }, + "reachability": { + "graphs": [ + { + "kind": "static", + "analyzer": "scanner.java@10.2.0", + "hash": "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "hashAlg": "blake3-256", + "casUri": "cas://reachability/graphs/blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2" + } + ], + "runtimeTraces": [], + "code_id_coverage": { + "total_nodes": 100, + "nodes_with_symbol_id": 100, + "nodes_with_code_id": 0, + "coverage_percent": 100.0 + } + }, + "outputs": {}, + "provenance": {} +} +``` + +**Expected canonical hash:** `sha256:e7f8a9b0...` (computed from canonical JSON) + +### 4.2 Manifest with Runtime Traces + +```json +{ + "schemaVersion": "2.0", + "scan": { + "id": "scan-test-002", + "time": "2025-12-13T11:00:00Z", + "mode": "record", + "scannerVersion": "10.2.0" + }, + "reachability": { + "graphs": [ + { + "kind": "static", + "analyzer": "scanner.java@10.2.0", + "hash": "blake3:1111111111111111111111111111111111111111111111111111111111111111", + "hashAlg": "blake3-256", + "casUri": "cas://reachability/graphs/blake3:1111111111111111111111111111111111111111111111111111111111111111" + } + ], + "runtimeTraces": [ + { + "source": "eventpipe", + "hash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "hashAlg": "sha256", + "casUri": "cas://reachability/runtime/sha256:2222222222222222222222222222222222222222222222222222222222222222", + "recordedAt": "2025-12-13T10:30:00Z" + } + ] + } +} +``` + +### 4.3 Sorting Validation Vector + +Input (unsorted): + +```json +{ + "reachability": { + "graphs": [ + {"casUri": "cas://reachability/graphs/blake3:zzzz...", "kind": "framework"}, + {"casUri": "cas://reachability/graphs/blake3:aaaa...", "kind": "static"} + ] + } +} +``` + +Expected output (sorted): + +```json +{ + "reachability": { + "graphs": [ + {"casUri": "cas://reachability/graphs/blake3:aaaa...", "kind": "static"}, + {"casUri": "cas://reachability/graphs/blake3:zzzz...", "kind": "framework"} + ] + } +} +``` + +### 4.4 Invalid Manifest Vectors + +| Test Case | Input | Expected Error | +|-----------|-------|----------------| +| Missing schemaVersion | `{}` | `REPLAY_MANIFEST_MISSING_VERSION` | +| Invalid version | `{"schemaVersion": "1.0"}` | `REPLAY_MANIFEST_VERSION_MISMATCH` (when v2 required) | +| Missing hashAlg | `{"hash": "blake3:..."}` | `REPLAY_MANIFEST_MISSING_HASH_ALG` | +| Unsorted graphs | See 4.3 input | `REPLAY_MANIFEST_UNSORTED_ENTRIES` | +| Missing CAS reference | `{"casUri": "cas://missing/..."}` | `REPLAY_MANIFEST_CAS_NOT_FOUND` | +| Hash mismatch | CAS content differs | `REPLAY_MANIFEST_HASH_MISMATCH` | + +--- + +## 5. Migration Path + +### 5.1 v1 → v2 Upgrade + +```csharp +public static ReplayManifest UpgradeToV2(ReplayManifest v1) +{ + return v1 with + { + SchemaVersion = "2.0", + Reachability = v1.Reachability with + { + Graphs = v1.Reachability.Graphs + .Select(g => g with { HashAlg = InferHashAlg(g.Hash) }) + .OrderBy(g => g.CasUri) + .ToList(), + RuntimeTraces = v1.Reachability.RuntimeTraces + .Select(t => t with { HashAlg = InferHashAlg(t.Hash) }) + .OrderBy(t => t.CasUri) + .ToList() + } + }; +} +``` + +### 5.2 Backward Compatibility + +- v2 readers MUST accept v1 manifests with warning +- v2 writers MUST always emit v2 format +- v1 writers deprecated after 2026-03-01 + +--- + +## 6. Test Fixture Locations + +``` +tests/Replay/ + fixtures/ + manifest-v2-minimal.json + manifest-v2-with-runtime.json + manifest-v2-sorted.json + manifest-v2-code-id-coverage.json + invalid/ + manifest-missing-version.json + manifest-unsorted.json + manifest-missing-hashalg.json + golden/ + manifest-v2-canonical.golden.json + manifest-v2-hash.golden.txt +``` + +--- + +## 7. Implementation Checklist + +- [ ] Update `ReplayManifest` record with v2 fields +- [ ] Add `hashAlg` to all hash-bearing types +- [ ] Implement sorting in `ReachabilityReplayWriter` +- [ ] Add CAS registration validation +- [ ] Create test fixtures +- [ ] Update `DETERMINISTIC_REPLAY.md` section 3 +- [ ] Wire into RecordModeService + +--- + +_Last updated: 2025-12-13. See Sprint 0401 GAP-REP-004 for implementation._ diff --git a/docs/uncertainty/README.md b/docs/uncertainty/README.md index f0e4823be..75ecdc234 100644 --- a/docs/uncertainty/README.md +++ b/docs/uncertainty/README.md @@ -224,5 +224,103 @@ Extended schema with tier information: - **Tier calculation:** `UncertaintyTierCalculator` in `src/Signals/StellaOps.Signals/Services/` - **Risk score math:** `ReachabilityScoringService.ComputeRiskScore()` (extend existing) -- **Policy integration:** `docs/reachability/policy-gate.md` for gate rules +- **Policy integration:** `docs/policy/dsl.md` §12 for uncertainty gates - **Lattice integration:** `docs/reachability/lattice.md` §9 for v1 lattice states + +--- + +## 8. Policy Guidance (v1 — Sprint 0401) + +Uncertainty gates enforce evidence-quality thresholds in the Policy Engine. When entropy is too high or evidence is missing, policies block or downgrade VEX decisions. + +### 8.1 Gate Mapping + +| Gate | Uncertainty State | Tier | Policy Action | +|------|------------------|------|---------------| +| `U1` | `MissingSymbolResolution` | T1/T2 | Block `not_affected`, require review | +| `U2` | `MissingPurl` | T2/T3 | Warn on `not_affected`, add review flag | +| `U3` | `UntrustedAdvisory` | T3/T4 | Advisory caveat, no blocking | + +### 8.2 Sample Policy Rules + +```dsl +// Block not_affected when symbol resolution has high entropy +rule u1_gate_high_entropy priority 5 { + when signals.uncertainty.level == "U1" + and signals.uncertainty.entropy >= 0.7 + then status := "under_investigation" + annotate gate := "U1" + annotate remediation := "Upload symbols or close unknowns registry" + because "High symbol entropy blocks strong VEX claims"; +} + +// Tier-based compound gate +rule tier1_block_not_affected priority 3 { + when signals.uncertainty.aggregateTier == "T1" + and vex.any(status == "not_affected") + then status := "under_investigation" + annotate blocked_reason := "T1 uncertainty requires evidence" + because "Maximum uncertainty tier blocks all exclusion claims"; +} +``` + +### 8.3 YAML Configuration + +```yaml +uncertainty_gates: + u1_gate: + entropy_threshold: 0.7 + blocked_statuses: [not_affected] + fallback_status: under_investigation + remediation_hint: "Upload symbols or resolve unknowns" + u2_gate: + entropy_threshold: 0.4 + blocked_statuses: [not_affected] + warn_on_block: true + u3_gate: + entropy_threshold: 0.1 + annotate_caveat: true +``` + +See `docs/policy/dsl.md` §12 for complete gate rules and tier-aware compound patterns. + +--- + +## 9. Remediation Actions + +Each uncertainty state has recommended remediation steps: + +| State | Code | Remediation | CLI Command | +|-------|------|-------------|-------------| +| MissingSymbolResolution | `U1` | Upload debug symbols, resolve unknowns | `stella symbols ingest --path ` | +| MissingPurl | `U2` | Generate lockfile, verify package coordinates | `stella sbom refresh --resolve` | +| UntrustedAdvisory | `U3` | Cross-reference trusted sources | `stella advisory verify --source NVD,GHSA` | +| Unknown | `U4` | Run initial analysis | `stella scan --full` | + +### 9.1 Automated Remediation Flow + +``` +1. Policy blocks decision with U1/U2 gate + ↓ +2. Console/CLI shows remediation hint + ↓ +3. User runs remediation command (e.g., stella symbols ingest) + ↓ +4. Signals recomputes uncertainty states + ↓ +5. Risk score updates, tier may drop + ↓ +6. Policy re-evaluates, decision may proceed +``` + +### 9.2 Remediation Priority + +When multiple uncertainty states exist, prioritize by tier: + +1. **T1 states first** — Block all exclusions until resolved +2. **T2 states** — May proceed with warnings if T1 cleared +3. **T3/T4 states** — Normal flow with caveats + +--- + +*Last updated: 2025-12-13 (Sprint 0401).* diff --git a/ops/crypto/sim-crypto-service/SimCryptoService.csproj b/ops/crypto/sim-crypto-service/SimCryptoService.csproj index b123492b1..152d35cb5 100644 --- a/ops/crypto/sim-crypto-service/SimCryptoService.csproj +++ b/ops/crypto/sim-crypto-service/SimCryptoService.csproj @@ -4,7 +4,6 @@ enable enable preview - diff --git a/ops/crypto/sim-crypto-smoke/SimCryptoSmoke.csproj b/ops/crypto/sim-crypto-smoke/SimCryptoSmoke.csproj index 21071f45d..3c92d16ad 100644 --- a/ops/crypto/sim-crypto-smoke/SimCryptoSmoke.csproj +++ b/ops/crypto/sim-crypto-smoke/SimCryptoSmoke.csproj @@ -5,7 +5,6 @@ enable enable preview - diff --git a/ops/mongo/indices/reachability_store_indices.js b/ops/mongo/indices/reachability_store_indices.js new file mode 100644 index 000000000..6f1da6a60 --- /dev/null +++ b/ops/mongo/indices/reachability_store_indices.js @@ -0,0 +1,67 @@ +/** + * MongoDB indexes for the shared reachability store collections used by Signals/Policy/Scanner. + * Run with: mongosh stellaops_db < reachability_store_indices.js + * + * Collections: + * - func_nodes: canonical function nodes keyed by graph + symbol ID and joinable by (purl, symbolDigest) + * - call_edges: canonical call edges keyed by graph and joinable by (purl, symbolDigest) + * - cve_func_hits: per-subject mapping of CVE -> affected/reachable functions with evidence pointers + * + * Created: 2025-12-13 (SIG-STORE-401-016) + */ + +// Switch to the target database (override via --eval "var dbName='custom'" if needed) +const targetDb = typeof dbName !== 'undefined' ? dbName : 'stellaops'; +db = db.getSiblingDB(targetDb); + +print(`Creating reachability store indexes on ${targetDb}...`); + +print(`- func_nodes`); +db.func_nodes.createIndex( + { "graphHash": 1, "symbolId": 1 }, + { name: "func_nodes_by_graph_symbol", unique: true, background: true } +); +db.func_nodes.createIndex( + { "purl": 1, "symbolDigest": 1 }, + { name: "func_nodes_by_purl_symboldigest", background: true, sparse: true } +); +db.func_nodes.createIndex( + { "codeId": 1 }, + { name: "func_nodes_by_code_id", background: true, sparse: true } +); + +print(`- call_edges`); +db.call_edges.createIndex( + { "graphHash": 1, "sourceId": 1, "targetId": 1, "type": 1 }, + { name: "call_edges_by_graph_edge", unique: true, background: true } +); +db.call_edges.createIndex( + { "graphHash": 1, "sourceId": 1 }, + { name: "call_edges_by_graph_source", background: true } +); +db.call_edges.createIndex( + { "graphHash": 1, "targetId": 1 }, + { name: "call_edges_by_graph_target", background: true } +); +db.call_edges.createIndex( + { "purl": 1, "symbolDigest": 1 }, + { name: "call_edges_by_purl_symboldigest", background: true, sparse: true } +); + +print(`- cve_func_hits`); +db.cve_func_hits.createIndex( + { "subjectKey": 1, "cveId": 1 }, + { name: "cve_func_hits_by_subject_cve", background: true } +); +db.cve_func_hits.createIndex( + { "cveId": 1, "purl": 1, "symbolDigest": 1 }, + { name: "cve_func_hits_by_cve_purl_symboldigest", background: true, sparse: true } +); +db.cve_func_hits.createIndex( + { "graphHash": 1 }, + { name: "cve_func_hits_by_graph", background: true, sparse: true } +); + +print("\nReachability store indexes created successfully."); +print("Run db.func_nodes.getIndexes(), db.call_edges.getIndexes(), db.cve_func_hits.getIndexes() to verify."); + diff --git a/samples/reachability/README.md b/samples/reachability/README.md new file mode 100644 index 000000000..faf0b85ae --- /dev/null +++ b/samples/reachability/README.md @@ -0,0 +1,38 @@ +# Reachability Evidence Samples + +This directory contains sample payloads for reachability evidence chain documentation. + +## Contents + +| File | Description | +|------|-------------| +| `richgraph-v1-sample.json` | Sample richgraph-v1 callgraph with `code_id`, `symbol_id`, and `graph_hash` | +| `openvex-affected-sample.json` | OpenVEX statement with `stellaops:reachability` extension for affected status | +| `openvex-not-affected-sample.json` | OpenVEX statement with unreachability evidence for not_affected status | +| `replay-manifest-v2-sample.json` | Replay manifest v2 with BLAKE3 hashes and `code_id_coverage` | +| `runtime-facts-sample.ndjson` | Runtime observation events in NDJSON format | + +## Usage + +These samples demonstrate the function-level evidence chain described in: +- `docs/reachability/function-level-evidence.md` +- `docs/api/signals/reachability-contract.md` +- `docs/contracts/richgraph-v1.md` + +## Verification + +Validate a richgraph-v1 sample: + +```bash +# Compute graph hash +stella graph verify --graph ./richgraph-v1-sample.json + +# Verify against manifest +stella replay verify --manifest ./replay-manifest-v2-sample.json --verbose +``` + +## Schema References + +- richgraph-v1: `docs/contracts/richgraph-v1.md` +- OpenVEX: https://openvex.dev/spec/v0.2.0 +- Replay manifest: `docs/reachability/function-level-evidence.md#6-replay-manifest-v2` diff --git a/samples/reachability/openvex-affected-sample.json b/samples/reachability/openvex-affected-sample.json new file mode 100644 index 000000000..1dd2a31c7 --- /dev/null +++ b/samples/reachability/openvex-affected-sample.json @@ -0,0 +1,86 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://stellaops.example/vex/2025-12-13/CVE-2021-44228-affected", + "author": "StellaOps Policy Engine", + "role": "automated-scanner", + "timestamp": "2025-12-13T10:00:00Z", + "version": 1, + "tooling": "StellaOps/1.0.0", + "statements": [ + { + "vulnerability": { + "@id": "CVE-2021-44228", + "name": "CVE-2021-44228", + "description": "Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints." + }, + "products": [ + { + "@id": "pkg:oci/myapp@sha256:abc123def456789012345678901234567890123456789012345678901234abcd", + "identifiers": { + "purl": "pkg:oci/myapp@sha256:abc123def456789012345678901234567890123456789012345678901234abcd" + }, + "subcomponents": [ + { + "@id": "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1", + "identifiers": { + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1" + } + } + ] + } + ], + "status": "affected", + "justification": "vulnerable_code_in_container", + "impact_statement": "Vulnerable Log4j error() method is reachable from main entry point via processRequest(). Runtime probes confirm 47 invocations observed.", + "action_statement": "Upgrade to log4j 2.17.1 or later. As a workaround, set log4j2.formatMsgNoLookups=true.", + "stellaops:reachability": { + "state": "CR", + "state_description": "ConfirmedReachable", + "confidence": 0.92, + "graph_hash": "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "graph_cas_uri": "cas://reachability/graphs/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "dsse_uri": "cas://reachability/graphs/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2.dsse", + "path": [ + { + "symbol_id": "sym:java:bWFpbi0xMjM0NTY3ODkwYWJjZGVm", + "code_id": "code:java:Y29kZS1tYWluLTEyMzQ1Njc4OTBhYmM", + "display": "com.example.app.Main.main(String[])", + "purl": "pkg:maven/com.example/app@1.0.0" + }, + { + "symbol_id": "sym:java:cHJvY2Vzc1JlcXVlc3QtYWJjZGVm", + "code_id": "code:java:Y29kZS1wcm9jZXNzLWFiY2RlZjEy", + "display": "com.example.app.RequestHandler.processRequest(HttpRequest)", + "purl": "pkg:maven/com.example/app@1.0.0" + }, + { + "symbol_id": "sym:java:bG9nRXJyb3ItMTIzNDU2Nzg5MGFiY2Q", + "code_id": "code:java:Y29kZS1sb2ctMTIzNDU2Nzg5MGFiY2Q", + "display": "org.apache.logging.log4j.Logger.error(String, Object...)", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1" + } + ], + "path_length": 3, + "evidence": { + "static": { + "graph_hash": "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "path_length": 3, + "confidence": 0.92 + }, + "runtime": { + "probe_id": "probe:jfr:scan-123-001", + "hit_count": 47, + "observed_at": "2025-12-13T09:45:00Z", + "observation_window": "24h" + } + }, + "fact_digest": "sha256:e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6", + "fact_version": 3, + "analyzer": { + "name": "scanner.java", + "version": "1.2.0" + } + } + } + ] +} diff --git a/samples/reachability/openvex-not-affected-sample.json b/samples/reachability/openvex-not-affected-sample.json new file mode 100644 index 000000000..9beb1bdfb --- /dev/null +++ b/samples/reachability/openvex-not-affected-sample.json @@ -0,0 +1,68 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://stellaops.example/vex/2025-12-13/CVE-2023-XXXXX-not-affected", + "author": "StellaOps Policy Engine", + "role": "automated-scanner", + "timestamp": "2025-12-13T10:00:00Z", + "version": 1, + "tooling": "StellaOps/1.0.0", + "statements": [ + { + "vulnerability": { + "@id": "CVE-2023-XXXXX", + "name": "CVE-2023-XXXXX", + "description": "Example vulnerability in deprecated API." + }, + "products": [ + { + "@id": "pkg:oci/myapp@sha256:abc123def456789012345678901234567890123456789012345678901234abcd", + "identifiers": { + "purl": "pkg:oci/myapp@sha256:abc123def456789012345678901234567890123456789012345678901234abcd" + }, + "subcomponents": [ + { + "@id": "pkg:maven/com.example/deprecated-lib@1.0.0", + "identifiers": { + "purl": "pkg:maven/com.example/deprecated-lib@1.0.0" + } + } + ] + } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "The deprecated API containing the vulnerable code path is not reachable from any entry point. Static analysis found no paths, and runtime probes observed zero invocations over 72 hours.", + "stellaops:reachability": { + "state": "CU", + "state_description": "ConfirmedUnreachable", + "confidence": 0.88, + "graph_hash": "blake3:d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5", + "graph_cas_uri": "cas://reachability/graphs/d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5", + "dsse_uri": "cas://reachability/graphs/d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5.dsse", + "path": [], + "path_length": 0, + "evidence": { + "static": { + "graph_hash": "blake3:d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5", + "path_length": 0, + "confidence": 0.85, + "analysis_note": "No path found from any root to vulnerable symbol" + }, + "runtime": { + "probe_id": "probe:jfr:scan-456-001", + "hit_count": 0, + "observed_at": "2025-12-13T09:45:00Z", + "observation_window": "72h", + "analysis_note": "Zero invocations observed during 72-hour monitoring window" + } + }, + "fact_digest": "sha256:f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7", + "fact_version": 2, + "analyzer": { + "name": "scanner.java", + "version": "1.2.0" + } + } + } + ] +} diff --git a/samples/reachability/replay-manifest-v2-sample.json b/samples/reachability/replay-manifest-v2-sample.json new file mode 100644 index 000000000..1b3d55433 --- /dev/null +++ b/samples/reachability/replay-manifest-v2-sample.json @@ -0,0 +1,62 @@ +{ + "schema": "stellaops.replay.manifest@v2", + "subject": "scan:myapp-123", + "generatedAt": "2025-12-13T10:00:00Z", + "hashAlg": "blake3", + "artifacts": [ + { + "kind": "richgraph", + "uri": "cas://reachability/graphs/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "hash": "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "dsseUri": "cas://reachability/graphs/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2.dsse", + "size": 24576, + "mediaType": "application/json" + }, + { + "kind": "runtime-facts", + "uri": "cas://reachability/runtime/sha256:xyz789abc123def456789012345678901234567890123456789012345678901234", + "hash": "sha256:xyz789abc123def456789012345678901234567890123456789012345678901234", + "size": 8192, + "mediaType": "application/x-ndjson" + }, + { + "kind": "sbom", + "uri": "cas://scanner-artifacts/scan-myapp-123/sbom.cdx.json", + "hash": "sha256:def456abc789012345678901234567890123456789012345678901234567890123", + "size": 102400, + "mediaType": "application/vnd.cyclonedx+json" + }, + { + "kind": "reachability-fact", + "uri": "cas://signals/facts/scan:myapp-123:pkg:maven/log4j:2.14.1:CVE-2021-44228", + "hash": "sha256:e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6", + "size": 4096, + "mediaType": "application/json" + } + ], + "analyzer": { + "name": "scanner.java", + "version": "1.2.0", + "toolchain_digest": "sha256:7b9e8c6d5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b1a0f9e8d" + }, + "code_id_coverage": { + "total_symbols": 1247, + "with_code_id": 1189, + "with_symbol_id": 1247, + "stripped_symbols": 58, + "coverage_pct": 95.3 + }, + "provenance": { + "scanner_version": "1.2.0", + "image_digest": "sha256:abc123def456789012345678901234567890123456789012345678901234abcd", + "scan_started_at": "2025-12-13T09:30:00Z", + "scan_completed_at": "2025-12-13T09:45:00Z", + "runtime_observation_window": "24h", + "runtime_probes_active": true + }, + "determinism": { + "manifest_hash": "blake3:fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210", + "reproducible": true, + "verified_at": "2025-12-13T10:00:00Z" + } +} diff --git a/samples/reachability/richgraph-v1-sample.json b/samples/reachability/richgraph-v1-sample.json new file mode 100644 index 000000000..5d502b685 --- /dev/null +++ b/samples/reachability/richgraph-v1-sample.json @@ -0,0 +1,117 @@ +{ + "schema": "richgraph-v1", + "analyzer": { + "name": "scanner.java", + "version": "1.2.0", + "toolchain_digest": "sha256:7b9e8c6d5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d" + }, + "nodes": [ + { + "id": "sym:java:bWFpbi0xMjM0NTY3ODkwYWJjZGVm", + "symbol_id": "sym:java:bWFpbi0xMjM0NTY3ODkwYWJjZGVm", + "code_id": "code:java:Y29kZS1tYWluLTEyMzQ1Njc4OTBhYmM", + "lang": "java", + "kind": "method", + "display": "com.example.app.Main.main(String[])", + "purl": "pkg:maven/com.example/app@1.0.0", + "build_id": null, + "symbol_digest": "sha256:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + "symbol": { + "mangled": null, + "demangled": "com.example.app.Main.main(String[])", + "source": "DWARF", + "confidence": 1.0 + }, + "evidence": ["bytecode"], + "attributes": {} + }, + { + "id": "sym:java:cHJvY2Vzc1JlcXVlc3QtYWJjZGVm", + "symbol_id": "sym:java:cHJvY2Vzc1JlcXVlc3QtYWJjZGVm", + "code_id": "code:java:Y29kZS1wcm9jZXNzLWFiY2RlZjEy", + "lang": "java", + "kind": "method", + "display": "com.example.app.RequestHandler.processRequest(HttpRequest)", + "purl": "pkg:maven/com.example/app@1.0.0", + "build_id": null, + "symbol_digest": "sha256:b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3", + "symbol": { + "mangled": null, + "demangled": "com.example.app.RequestHandler.processRequest(HttpRequest)", + "source": "DWARF", + "confidence": 0.98 + }, + "evidence": ["bytecode", "import"], + "attributes": {} + }, + { + "id": "sym:java:bG9nRXJyb3ItMTIzNDU2Nzg5MGFiY2Q", + "symbol_id": "sym:java:bG9nRXJyb3ItMTIzNDU2Nzg5MGFiY2Q", + "code_id": "code:java:Y29kZS1sb2ctMTIzNDU2Nzg5MGFiY2Q", + "lang": "java", + "kind": "method", + "display": "org.apache.logging.log4j.Logger.error(String, Object...)", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1", + "build_id": null, + "symbol_digest": "sha256:c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4", + "symbol": { + "mangled": null, + "demangled": "org.apache.logging.log4j.Logger.error(String, Object...)", + "source": "DWARF", + "confidence": 0.95 + }, + "evidence": ["bytecode", "import"], + "attributes": { + "vulnerable": "CVE-2021-44228" + } + }, + { + "id": "sym:java:dW51c2VkTWV0aG9kLWFiY2RlZjEyMzQ", + "symbol_id": "sym:java:dW51c2VkTWV0aG9kLWFiY2RlZjEyMzQ", + "code_id": "code:java:Y29kZS11bnVzZWQtYWJjZGVmMTIzNA", + "lang": "java", + "kind": "method", + "display": "com.example.app.Unused.unusedMethod()", + "purl": "pkg:maven/com.example/app@1.0.0", + "build_id": null, + "symbol_digest": "sha256:d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5", + "symbol": { + "mangled": null, + "demangled": "com.example.app.Unused.unusedMethod()", + "source": "DWARF", + "confidence": 0.92 + }, + "evidence": ["bytecode"], + "attributes": {} + } + ], + "edges": [ + { + "from": "sym:java:bWFpbi0xMjM0NTY3ODkwYWJjZGVm", + "to": "sym:java:cHJvY2Vzc1JlcXVlc3QtYWJjZGVm", + "kind": "call", + "purl": "pkg:maven/com.example/app@1.0.0", + "symbol_digest": "sha256:b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3", + "confidence": 1.0, + "evidence": ["bytecode"], + "candidates": [] + }, + { + "from": "sym:java:cHJvY2Vzc1JlcXVlc3QtYWJjZGVm", + "to": "sym:java:bG9nRXJyb3ItMTIzNDU2Nzg5MGFiY2Q", + "kind": "virtual", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1", + "symbol_digest": "sha256:c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4", + "confidence": 0.92, + "evidence": ["bytecode", "import"], + "candidates": [] + } + ], + "roots": [ + { + "id": "sym:java:bWFpbi0xMjM0NTY3ODkwYWJjZGVm", + "phase": "runtime", + "source": "main" + } + ] +} diff --git a/samples/reachability/runtime-facts-sample.ndjson b/samples/reachability/runtime-facts-sample.ndjson new file mode 100644 index 000000000..d4835802e --- /dev/null +++ b/samples/reachability/runtime-facts-sample.ndjson @@ -0,0 +1,3 @@ +{"symbolId":"sym:java:bWFpbi0xMjM0NTY3ODkwYWJjZGVm","codeId":"code:java:Y29kZS1tYWluLTEyMzQ1Njc4OTBhYmM","hitCount":1,"loaderBase":"0x7f1234560000","processId":12345,"processName":"java","containerId":"abc123def456","observedAt":"2025-12-13T09:30:00Z"} +{"symbolId":"sym:java:cHJvY2Vzc1JlcXVlc3QtYWJjZGVm","codeId":"code:java:Y29kZS1wcm9jZXNzLWFiY2RlZjEy","hitCount":47,"loaderBase":"0x7f1234560000","processId":12345,"processName":"java","containerId":"abc123def456","observedAt":"2025-12-13T09:45:00Z"} +{"symbolId":"sym:java:bG9nRXJyb3ItMTIzNDU2Nzg5MGFiY2Q","codeId":"code:java:Y29kZS1sb2ctMTIzNDU2Nzg5MGFiY2Q","hitCount":47,"loaderBase":"0x7f1234789000","processId":12345,"processName":"java","containerId":"abc123def456","observedAt":"2025-12-13T09:45:00Z"} diff --git a/samples/runtime/java-fat-archive/libs/app-fat.jar b/samples/runtime/java-fat-archive/libs/app-fat.jar new file mode 100644 index 0000000000000000000000000000000000000000..9579c757d7716515d123800bcfe116fb3b92d9b2 GIT binary patch literal 1092 zcmbtTF>ljA6h4!JkR@;35YizeHlN!L9U^5&tE4ii(@2RB>q~qICidO+d<{tL(196= zpTJK*VyJ|m6B{Fc0ttzk1$cIf>s+f2VC$XFclW;czW2Qw^crjHgpj*v51xM83coCH zc8Io#7Ij8#KkRpd3>I7j6PYpo5;B$Y0F?5lU;>3}N=wDHNqFuYlGnd~d^mqdhvhN4 zy6w3$EssohMSXHrWQ-@LT~IZ?%=8T`-=jU^0hUB4(}U zpgni6LgHlY?ffGqe}-02t4Kuce%S2{M=a_(XJoI)GXjxJd2akC%WzGK@jN$zMEJWY z@nzlI1-a*95V@v+GJc9FqphQN=(X|w94Ds zt_Ll`UfCA0)J@u1mYnWNtvuJJ`}5K8&CnsFk9HsF?i?J9O3l(_9Pn(+c51qV_&Mj#Y#O&^*a|_=zcQ3rUjf^ z>!sYwPPSPgvD))*90$GIcM1>(-x$ANOT { + return import("multi"); +} diff --git a/samples/runtime/node-detection-gaps/packages/lib/package.json b/samples/runtime/node-detection-gaps/packages/lib/package.json new file mode 100644 index 000000000..24e620e8d --- /dev/null +++ b/samples/runtime/node-detection-gaps/packages/lib/package.json @@ -0,0 +1,4 @@ +{ + "name": "lib", + "version": "2.0.0" +} diff --git a/samples/runtime/node-detection-gaps/packages/lib/src/index.ts b/samples/runtime/node-detection-gaps/packages/lib/src/index.ts new file mode 100644 index 000000000..b874413a3 --- /dev/null +++ b/samples/runtime/node-detection-gaps/packages/lib/src/index.ts @@ -0,0 +1 @@ +export const libValue = 41; diff --git a/samples/runtime/node-detection-gaps/yarn.lock b/samples/runtime/node-detection-gaps/yarn.lock new file mode 100644 index 000000000..cf84e0890 --- /dev/null +++ b/samples/runtime/node-detection-gaps/yarn.lock @@ -0,0 +1,13 @@ +__metadata: + version: 8 + cacheKey: 10 + +"multi@npm:^1.0.0": + version: "1.0.0" + resolution: "multi@npm:1.0.0" + checksum: "abcd1234" + +"multi@npm:^2.0.0": + version: "2.0.0" + resolution: "multi@npm:2.0.0" + integrity: "sha512-xyz987" diff --git a/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs b/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs index cf6981ffd..3c2fa4009 100644 --- a/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs +++ b/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs @@ -13,6 +13,7 @@ using StellaOps.Attestor.Core.Storage; using StellaOps.Attestor.Core.Submission; using StellaOps.Attestor.Core.Transparency; using StellaOps.Attestor.Core.Verification; +using StellaOps.Attestor.Core.Bulk; using StellaOps.Attestor.Infrastructure.Rekor; using StellaOps.Attestor.Infrastructure.Storage; using StellaOps.Attestor.Infrastructure.Submission; diff --git a/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj b/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj index 5cd601ab7..d11333b24 100644 --- a/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj +++ b/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/MessagingAttestorVerificationCache.cs b/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/MessagingAttestorVerificationCache.cs new file mode 100644 index 000000000..b54dc757f --- /dev/null +++ b/src/Attestor/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/MessagingAttestorVerificationCache.cs @@ -0,0 +1,107 @@ +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Verification; +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; + +namespace StellaOps.Attestor.Infrastructure.Verification; + +/// +/// Attestor verification cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +internal sealed class MessagingAttestorVerificationCache : IAttestorVerificationCache +{ + private readonly IDistributedCache _cache; + private readonly ILogger _logger; + private readonly TimeSpan _ttl; + + public MessagingAttestorVerificationCache( + IDistributedCacheFactory cacheFactory, + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + ArgumentNullException.ThrowIfNull(options); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var ttlSeconds = Math.Max(1, options.Value.Cache.Verification.TtlSeconds); + _ttl = TimeSpan.FromSeconds(ttlSeconds); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = "attestor:verify:", + DefaultTtl = _ttl, + }); + + _logger.LogInformation( + "Initialized MessagingAttestorVerificationCache with provider {Provider}, TTL {Ttl}s", + _cache.ProviderName, + _ttl.TotalSeconds.ToString(CultureInfo.InvariantCulture)); + } + + public async Task GetAsync( + string subject, + string envelopeId, + string policyVersion, + CancellationToken cancellationToken = default) + { + var cacheKey = BuildCacheKey(subject, envelopeId, policyVersion); + var result = await _cache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false); + + if (result.HasValue) + { + return result.Value; + } + + return null; + } + + public async Task SetAsync( + string subject, + string envelopeId, + string policyVersion, + AttestorVerificationResult result, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(result); + + var cacheKey = BuildCacheKey(subject, envelopeId, policyVersion); + var entryOptions = new CacheEntryOptions { TimeToLive = _ttl }; + + await _cache.SetAsync(cacheKey, result, entryOptions, cancellationToken).ConfigureAwait(false); + + var subjectKey = Normalize(subject); + _logger.LogDebug( + "Cached verification result for subject {Subject} envelope {Envelope} policy {Policy} with TTL {TtlSeconds}s.", + subjectKey, + Normalize(envelopeId), + Normalize(policyVersion), + _ttl.TotalSeconds.ToString(CultureInfo.InvariantCulture)); + } + + public async Task InvalidateSubjectAsync(string subject, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(subject)) + { + return; + } + + var subjectKey = Normalize(subject); + // Pattern: attestor:verify:|* + var pattern = $"{subjectKey}|*"; + var count = await _cache.InvalidateByPatternAsync(pattern, cancellationToken).ConfigureAwait(false); + + _logger.LogDebug("Invalidated {Count} verification cache entries for subject {Subject}.", count, subjectKey); + } + + private static string BuildCacheKey(string subject, string envelopeId, string policyVersion) => + string.Concat(Normalize(subject), "|", Normalize(envelopeId), "|", Normalize(policyVersion)); + + private static string Normalize(string? value) => (value ?? string.Empty).Trim(); +} diff --git a/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/MessagingTokenCache.cs b/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/MessagingTokenCache.cs new file mode 100644 index 000000000..a26d12f11 --- /dev/null +++ b/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/MessagingTokenCache.cs @@ -0,0 +1,83 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; + +namespace StellaOps.Auth.Client; + +/// +/// Token cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +public sealed class MessagingTokenCache : IStellaOpsTokenCache +{ + private readonly IDistributedCache _cache; + private readonly TimeProvider _timeProvider; + private readonly Func _normalizer; + private readonly TimeSpan _expirationSkew; + + public MessagingTokenCache( + IDistributedCacheFactory cacheFactory, + TimeProvider? timeProvider = null, + TimeSpan? expirationSkew = null) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + + _timeProvider = timeProvider ?? TimeProvider.System; + _expirationSkew = expirationSkew ?? TimeSpan.FromSeconds(30); + _normalizer = static entry => entry.NormalizeScopes(); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = "auth:token:", + }); + } + + public async ValueTask GetAsync(string key, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + + var result = await _cache.GetAsync(key, cancellationToken).ConfigureAwait(false); + + if (!result.HasValue) + { + return null; + } + + var entry = result.Value; + + // Check if expired with skew + if (entry.IsExpired(_timeProvider, _expirationSkew)) + { + await _cache.InvalidateAsync(key, cancellationToken).ConfigureAwait(false); + return null; + } + + return entry; + } + + public async ValueTask SetAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(entry); + + var normalizedEntry = _normalizer(entry); + var now = _timeProvider.GetUtcNow(); + var ttl = normalizedEntry.ExpiresAtUtc - now; + + if (ttl <= TimeSpan.Zero) + { + return; + } + + var entryOptions = new CacheEntryOptions { TimeToLive = ttl }; + await _cache.SetAsync(key, normalizedEntry, entryOptions, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask RemoveAsync(string key, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + await _cache.InvalidateAsync(key, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj b/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj index 1e0866f03..4fb788164 100644 --- a/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj +++ b/src/Authority/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj @@ -30,6 +30,7 @@ + diff --git a/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/Claims/MessagingLdapClaimsCache.cs b/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/Claims/MessagingLdapClaimsCache.cs new file mode 100644 index 000000000..a2be9cf1d --- /dev/null +++ b/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/Claims/MessagingLdapClaimsCache.cs @@ -0,0 +1,57 @@ +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; + +namespace StellaOps.Authority.Plugin.Ldap.Claims; + +/// +/// LDAP claims cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +internal sealed class MessagingLdapClaimsCache : ILdapClaimsCache +{ + private readonly IDistributedCache _cache; + private readonly string _pluginName; + private readonly TimeSpan _ttl; + + public MessagingLdapClaimsCache( + IDistributedCacheFactory cacheFactory, + string pluginName, + LdapClaimsCacheOptions options) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + ArgumentException.ThrowIfNullOrWhiteSpace(pluginName); + ArgumentNullException.ThrowIfNull(options); + + _pluginName = pluginName; + _ttl = TimeSpan.FromSeconds(options.TtlSeconds); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = $"ldap:claims:{pluginName}:", + DefaultTtl = _ttl, + }); + } + + public async ValueTask GetAsync(string subjectId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(subjectId); + + var key = BuildKey(subjectId); + var result = await _cache.GetAsync(key, cancellationToken).ConfigureAwait(false); + + return result.HasValue ? result.Value : null; + } + + public async ValueTask SetAsync(string subjectId, LdapCachedClaims claims, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(subjectId); + ArgumentNullException.ThrowIfNull(claims); + + var key = BuildKey(subjectId); + var entryOptions = new CacheEntryOptions { TimeToLive = _ttl }; + + await _cache.SetAsync(key, claims, entryOptions, cancellationToken).ConfigureAwait(false); + } + + private string BuildKey(string subjectId) => subjectId.ToLowerInvariant(); +} diff --git a/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/StellaOps.Authority.Plugin.Ldap.csproj b/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/StellaOps.Authority.Plugin.Ldap.csproj index aacd3d7b7..5906ef5f0 100644 --- a/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/StellaOps.Authority.Plugin.Ldap.csproj +++ b/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Ldap/StellaOps.Authority.Plugin.Ldap.csproj @@ -13,7 +13,7 @@ - + @@ -21,5 +21,6 @@ + diff --git a/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj b/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj index e688b251f..c422425ec 100644 --- a/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj +++ b/src/Authority/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj @@ -9,7 +9,5 @@ - - - + diff --git a/src/Authority/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj b/src/Authority/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj index 0c5e73c05..05e546b83 100644 --- a/src/Authority/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj +++ b/src/Authority/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj @@ -11,9 +11,7 @@ - - - + diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/README.md b/src/Bench/StellaOps.Bench/Scanner.Analyzers/README.md index 26021639a..c2f253c6f 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/README.md +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/README.md @@ -8,9 +8,16 @@ The bench harness exercises the language analyzers against representative filesy - `baseline.csv` – Reference numbers captured on the 4 vCPU warm rig described in `docs/12_PERFORMANCE_WORKBOOK.md`. CI publishes fresh CSVs so perf trends stay visible. ## Current scenarios +- `node_detection_gaps_fixture` - runs the Node analyzer across `samples/runtime/node-detection-gaps` (workspaces + lock-only + import scan). - `node_monorepo_walk` → runs the Node analyzer across `samples/runtime/npm-monorepo`. - `java_demo_archive` → runs the Java analyzer against `samples/runtime/java-demo/libs/demo.jar`. -- `python_site_packages_walk` → temporary metadata walk over `samples/runtime/python-venv` until the Python analyzer lands. +- `python_site_packages_scan` → runs the Python analyzer across `samples/runtime/python-venv`. +- `python_pip_cache_fixture` → runs the Python analyzer across the RECORD-heavy pip cache fixture. +- `python_layered_editable_fixture` → runs the Python analyzer across layered/container-root layouts (`layers/`, `.layers/`, `layer*`). + +- `bun_multi_workspace_fixture` - runs the Bun analyzer across the Bun multi-workspace fixture under the Bun analyzer tests. + +See `config.json` for the authoritative list. ## Running locally diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/NodeBenchMetrics.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/NodeBenchMetrics.cs new file mode 100644 index 000000000..1ab4d2ffa --- /dev/null +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/NodeBenchMetrics.cs @@ -0,0 +1,268 @@ +using System.Collections.Generic; +using System.Linq; +using StellaOps.Scanner.Analyzers.Lang; + +namespace StellaOps.Bench.ScannerAnalyzers.Scenarios; + +internal static class NodeBenchMetrics +{ + private static readonly HashSet Extensions = new(StringComparer.OrdinalIgnoreCase) + { + ".js", + ".jsx", + ".mjs", + ".cjs", + ".ts", + ".tsx", + ".mts", + ".cts" + }; + + private static readonly string[] IgnoredDirectories = + { + ".bin", + ".cache", + ".store", + "__pycache__" + }; + + public static IReadOnlyDictionary Compute(string rootPath, LanguageAnalyzerResult result) + { + ArgumentException.ThrowIfNullOrWhiteSpace(rootPath); + ArgumentNullException.ThrowIfNull(result); + + var scanRoots = CollectImportScanRoots(rootPath, result); + var packagesScanned = 0; + var filesScanned = 0; + long bytesScanned = 0; + var cappedPackages = 0; + + foreach (var scanRoot in scanRoots) + { + var counters = CountImportScan(scanRoot); + packagesScanned++; + filesScanned += counters.FilesScanned; + bytesScanned += counters.BytesScanned; + + if (counters.Capped) + { + cappedPackages++; + } + } + + return new SortedDictionary(StringComparer.Ordinal) + { + ["node.importScan.packages"] = packagesScanned, + ["node.importScan.filesScanned"] = filesScanned, + ["node.importScan.bytesScanned"] = bytesScanned, + ["node.importScan.cappedPackages"] = cappedPackages + }; + } + + public static bool AreEqual(IReadOnlyDictionary left, IReadOnlyDictionary right) + { + ArgumentNullException.ThrowIfNull(left); + ArgumentNullException.ThrowIfNull(right); + + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Count != right.Count) + { + return false; + } + + foreach (var (key, value) in left) + { + if (!right.TryGetValue(key, out var other) || other != value) + { + return false; + } + } + + return true; + } + + private static IReadOnlyList CollectImportScanRoots(string rootPath, LanguageAnalyzerResult result) + { + var comparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var roots = new HashSet(comparer); + + var fullRoot = Path.GetFullPath(rootPath); + + foreach (var record in result.Components) + { + if (!string.Equals(record.AnalyzerId, "node", StringComparison.Ordinal)) + { + continue; + } + + if (!record.Metadata.TryGetValue("path", out var relativePath) || string.IsNullOrWhiteSpace(relativePath)) + { + continue; + } + + var isRoot = string.Equals(relativePath, ".", StringComparison.Ordinal); + var isWorkspaceMember = record.Metadata.TryGetValue("workspaceMember", out var workspaceMember) + && string.Equals(workspaceMember, "true", StringComparison.OrdinalIgnoreCase); + + if (!isRoot && !isWorkspaceMember) + { + continue; + } + + var absolute = isRoot + ? fullRoot + : Path.GetFullPath(Path.Combine(fullRoot, relativePath.Replace('/', Path.DirectorySeparatorChar))); + + if (Directory.Exists(absolute)) + { + roots.Add(absolute); + } + } + + return roots.OrderBy(static p => p, StringComparer.Ordinal).ToArray(); + } + + private static ImportScanCounters CountImportScan(string rootPath) + { + const int maxFilesPerPackage = 500; + const long maxBytesPerPackage = 5L * 1024 * 1024; + const long maxFileBytes = 512L * 1024; + const int maxDepth = 20; + + var filesScanned = 0; + long bytesScanned = 0; + var capped = false; + + foreach (var file in EnumerateSourceFiles(rootPath, maxDepth)) + { + if (filesScanned >= maxFilesPerPackage || bytesScanned >= maxBytesPerPackage) + { + capped = true; + break; + } + + long length; + try + { + length = new FileInfo(file).Length; + } + catch + { + continue; + } + + if (length <= 0 || length > maxFileBytes) + { + continue; + } + + if (bytesScanned + length > maxBytesPerPackage) + { + capped = true; + break; + } + + bytesScanned += length; + filesScanned++; + } + + return new ImportScanCounters(filesScanned, bytesScanned, capped); + } + + private static IEnumerable EnumerateSourceFiles(string root, int maxDepth) + { + var pathComparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var stack = new Stack<(string Path, int Depth)>(); + stack.Push((root, 0)); + + while (stack.Count > 0) + { + var (current, depth) = stack.Pop(); + + IEnumerable files; + try + { + files = Directory.EnumerateFiles(current, "*", SearchOption.TopDirectoryOnly); + } + catch + { + files = Array.Empty(); + } + + foreach (var file in files.OrderBy(static f => f, pathComparer)) + { + var ext = Path.GetExtension(file); + if (!string.IsNullOrWhiteSpace(ext) && Extensions.Contains(ext)) + { + yield return file; + } + } + + if (depth >= maxDepth) + { + continue; + } + + IEnumerable dirs; + try + { + dirs = Directory.EnumerateDirectories(current, "*", SearchOption.TopDirectoryOnly); + } + catch + { + dirs = Array.Empty(); + } + + var ordered = dirs + .Where(static d => !ShouldSkipImportDirectory(Path.GetFileName(d))) + .OrderBy(static d => d, pathComparer) + .ToArray(); + + for (var i = ordered.Length - 1; i >= 0; i--) + { + stack.Push((ordered[i], depth + 1)); + } + } + } + + private static bool ShouldSkipImportDirectory(string? name) + { + if (string.IsNullOrWhiteSpace(name)) + { + return true; + } + + if (string.Equals(name, "node_modules", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return ShouldSkipDirectory(name); + } + + private static bool ShouldSkipDirectory(string name) + { + if (name.Length == 0) + { + return true; + } + + if (name[0] == '.') + { + return !string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase); + } + + return IgnoredDirectories.Any(ignored => string.Equals(name, ignored, StringComparison.OrdinalIgnoreCase)); + } + + private readonly record struct ImportScanCounters(int FilesScanned, long BytesScanned, bool Capped); +} diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs index 0100de0b8..7bcc1386a 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Program.cs @@ -43,7 +43,10 @@ internal static class Program stats.P95Ms, stats.MaxMs, iterations, - scenarioThreshold); + scenarioThreshold) + { + Metrics = execution.Metrics + }; results.Add(result); @@ -101,7 +104,7 @@ internal static class Program } catch (Exception ex) { - Console.Error.WriteLine(ex.Message); + Console.Error.WriteLine(ex); return 1; } } diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs index 183415b6a..0c31e0597 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs @@ -53,6 +53,7 @@ internal static class BenchmarkJsonWriter report.Result.P95Ms, report.Result.MaxMs, report.Result.ThresholdMs, + report.Result.Metrics, baseline is null ? null : new BenchmarkJsonScenarioBaseline( @@ -84,6 +85,7 @@ internal static class BenchmarkJsonWriter double P95Ms, double MaxMs, double ThresholdMs, + IReadOnlyDictionary? Metrics, BenchmarkJsonScenarioBaseline? Baseline, BenchmarkJsonScenarioRegression Regression); diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs index 55ab4ba46..8ddd2c6fe 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkScenarioReport.cs @@ -1,3 +1,4 @@ +using System.Globalization; using StellaOps.Bench.ScannerAnalyzers.Baseline; namespace StellaOps.Bench.ScannerAnalyzers.Reporting; @@ -35,7 +36,13 @@ internal sealed class BenchmarkScenarioReport } var percentage = (MaxRegressionRatio.Value - 1d) * 100d; - return $"{Result.Id} exceeded regression budget: max {Result.MaxMs:F2} ms vs baseline {Baseline!.MaxMs:F2} ms (+{percentage:F1}%)"; + return string.Format( + CultureInfo.InvariantCulture, + "{0} exceeded regression budget: max {1:F2} ms vs baseline {2:F2} ms (+{3:F1}%)", + Result.Id, + Result.MaxMs, + Baseline!.MaxMs, + percentage); } private static double? CalculateRatio(double current, double? baseline) diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs index 03697ff59..dca03dbe7 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs @@ -1,4 +1,5 @@ using System.Globalization; +using System.Linq; using System.Text; namespace StellaOps.Bench.ScannerAnalyzers.Reporting; @@ -20,6 +21,10 @@ internal static class PrometheusWriter var builder = new StringBuilder(); builder.AppendLine("# HELP scanner_analyzer_bench_duration_ms Analyzer benchmark duration metrics in milliseconds."); builder.AppendLine("# TYPE scanner_analyzer_bench_duration_ms gauge"); + builder.AppendLine("# HELP scanner_analyzer_bench_sample_count Analyzer benchmark sample counts (component/file counts)."); + builder.AppendLine("# TYPE scanner_analyzer_bench_sample_count gauge"); + builder.AppendLine("# HELP scanner_analyzer_bench_metric Additional analyzer benchmark metrics."); + builder.AppendLine("# TYPE scanner_analyzer_bench_metric gauge"); foreach (var report in reports) { @@ -28,6 +33,7 @@ internal static class PrometheusWriter AppendMetric(builder, "scanner_analyzer_bench_p95_ms", scenarioLabel, report.Result.P95Ms); AppendMetric(builder, "scanner_analyzer_bench_max_ms", scenarioLabel, report.Result.MaxMs); AppendMetric(builder, "scanner_analyzer_bench_threshold_ms", scenarioLabel, report.Result.ThresholdMs); + AppendMetric(builder, "scanner_analyzer_bench_sample_count", scenarioLabel, report.Result.SampleCount); if (report.Baseline is { } baseline) { @@ -41,6 +47,19 @@ internal static class PrometheusWriter AppendMetric(builder, "scanner_analyzer_bench_regression_limit", scenarioLabel, report.RegressionLimit); AppendMetric(builder, "scanner_analyzer_bench_regression_breached", scenarioLabel, report.RegressionBreached ? 1 : 0); } + + if (report.Result.Metrics is { Count: > 0 } metrics) + { + foreach (var metric in metrics.OrderBy(static item => item.Key, StringComparer.Ordinal)) + { + builder.Append("scanner_analyzer_bench_metric{scenario=\""); + builder.Append(scenarioLabel); + builder.Append("\",name=\""); + builder.Append(Escape(metric.Key)); + builder.Append("\"} "); + builder.AppendLine(metric.Value.ToString("G17", CultureInfo.InvariantCulture)); + } + } } File.WriteAllText(resolved, builder.ToString(), Encoding.UTF8); diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs index 4632bb6c2..32fcb723c 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioResult.cs @@ -12,6 +12,8 @@ internal sealed record ScenarioResult( int Iterations, double ThresholdMs) { + public IReadOnlyDictionary? Metrics { get; init; } + public string IdColumn => Id.Length <= 28 ? Id.PadRight(28) : Id[..28]; public string SampleCountColumn => SampleCount.ToString(CultureInfo.InvariantCulture).PadLeft(5); diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs index bbbb4a3b0..aafd509a7 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text.Json; using System.Text.RegularExpressions; using StellaOps.Scanner.Analyzers.Lang; +using StellaOps.Scanner.Analyzers.Lang.Bun; using StellaOps.Scanner.Analyzers.Lang.Go; using StellaOps.Scanner.Analyzers.Lang.Java; using StellaOps.Scanner.Analyzers.Lang.Node; @@ -17,7 +18,7 @@ internal interface IScenarioRunner Task ExecuteAsync(string rootPath, int iterations, CancellationToken cancellationToken); } -internal sealed record ScenarioExecutionResult(double[] Durations, int SampleCount); +internal sealed record ScenarioExecutionResult(double[] Durations, int SampleCount, IReadOnlyDictionary? Metrics = null); internal static class ScenarioRunnerFactory { @@ -40,6 +41,7 @@ internal static class ScenarioRunnerFactory internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner { private readonly IReadOnlyList> _analyzerFactories; + private readonly bool _includesNodeAnalyzer; public LanguageAnalyzerScenarioRunner(IEnumerable analyzerIds) { @@ -48,11 +50,15 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner throw new ArgumentNullException(nameof(analyzerIds)); } - _analyzerFactories = analyzerIds + var normalizedIds = analyzerIds .Where(static id => !string.IsNullOrWhiteSpace(id)) - .Select(CreateFactory) + .Select(static id => id.Trim().ToLowerInvariant()) .ToArray(); + _includesNodeAnalyzer = normalizedIds.Contains("node", StringComparer.Ordinal); + + _analyzerFactories = normalizedIds.Select(CreateFactory).ToArray(); + if (_analyzerFactories.Count == 0) { throw new InvalidOperationException("At least one analyzer id must be provided."); @@ -70,6 +76,7 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner var engine = new LanguageAnalyzerEngine(analyzers); var durations = new double[iterations]; var componentCount = -1; + IReadOnlyDictionary? metrics = null; for (var i = 0; i < iterations; i++) { @@ -91,6 +98,19 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner { throw new InvalidOperationException($"Analyzer output count changed between iterations ({componentCount} vs {currentCount})."); } + + if (_includesNodeAnalyzer) + { + var currentMetrics = NodeBenchMetrics.Compute(rootPath, result); + if (metrics is null) + { + metrics = currentMetrics; + } + else if (!NodeBenchMetrics.AreEqual(metrics, currentMetrics)) + { + throw new InvalidOperationException($"Analyzer metrics changed between iterations for '{rootPath}'."); + } + } } if (componentCount < 0) @@ -98,7 +118,7 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner componentCount = 0; } - return new ScenarioExecutionResult(durations, componentCount); + return new ScenarioExecutionResult(durations, componentCount, metrics); } private static Func CreateFactory(string analyzerId) @@ -106,6 +126,7 @@ internal sealed class LanguageAnalyzerScenarioRunner : IScenarioRunner var id = analyzerId.Trim().ToLowerInvariant(); return id switch { + "bun" => static () => new BunLanguageAnalyzer(), "java" => static () => new JavaLanguageAnalyzer(), "go" => static () => new GoLanguageAnalyzer(), "node" => static () => new NodeLanguageAnalyzer(), diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj index 8c70a2730..4624e7418 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj @@ -11,6 +11,7 @@ + @@ -21,4 +22,4 @@ - \ No newline at end of file + diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/baseline.csv b/src/Bench/StellaOps.Bench/Scanner.Analyzers/baseline.csv index a75ee90e0..a1a1ea4c8 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/baseline.csv +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/baseline.csv @@ -1,7 +1,11 @@ scenario,iterations,sample_count,mean_ms,p95_ms,max_ms -node_monorepo_walk,5,4,6.0975,21.7421,26.8537 -java_demo_archive,5,1,6.2007,23.4837,29.1143 -go_buildinfo_fixture,5,2,6.1949,22.6851,27.9196 -dotnet_multirid_fixture,5,2,11.4884,37.7460,46.4850 -python_site_packages_scan,5,3,5.6420,18.2943,22.3739 -python_pip_cache_fixture,5,1,5.8598,13.2855,15.6256 +node_monorepo_walk,5,4,15.5399,50.3210,61.7146 +node_detection_gaps_fixture,5,5,31.8434,96.4542,117.3238 +java_demo_archive,5,1,13.6363,49.4627,61.3100 +java_fat_archive,5,2,3.5181,8.1467,9.4927 +go_buildinfo_fixture,5,2,6.9861,25.8818,32.1304 +dotnet_multirid_fixture,5,2,11.8266,38.9340,47.8401 +python_site_packages_scan,5,3,36.7930,105.6978,128.4211 +python_pip_cache_fixture,5,1,20.1829,30.9147,34.3257 +python_layered_editable_fixture,5,3,31.8757,39.7647,41.5656 +bun_multi_workspace_fixture,5,2,12.4463,45.1913,55.9832 diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json b/src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json index 456c32466..ec693933a 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json @@ -10,6 +10,15 @@ "node" ] }, + { + "id": "node_detection_gaps_fixture", + "label": "Node analyzer detection gaps fixture (workspace + lock-only + imports)", + "root": "samples/runtime/node-detection-gaps", + "analyzers": [ + "node" + ], + "thresholdMs": 2000 + }, { "id": "java_demo_archive", "label": "Java analyzer on demo jar", @@ -18,6 +27,15 @@ "java" ] }, + { + "id": "java_fat_archive", + "label": "Java analyzer on fat jar (embedded libs)", + "root": "samples/runtime/java-fat-archive", + "analyzers": [ + "java" + ], + "thresholdMs": 1000 + }, { "id": "go_buildinfo_fixture", "label": "Go analyzer on build-info binary", @@ -49,6 +67,24 @@ "analyzers": [ "python" ] + }, + { + "id": "python_layered_editable_fixture", + "label": "Python analyzer on layered/container roots fixture", + "root": "src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable", + "analyzers": [ + "python" + ], + "thresholdMs": 2000 + }, + { + "id": "bun_multi_workspace_fixture", + "label": "Bun analyzer on multi-workspace fixture", + "root": "src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace", + "analyzers": [ + "bun" + ], + "thresholdMs": 1000 } ] } diff --git a/src/Bench/StellaOps.Bench/Scanner.Analyzers/lang/README.md b/src/Bench/StellaOps.Bench/Scanner.Analyzers/lang/README.md index 722305165..6a68273cd 100644 --- a/src/Bench/StellaOps.Bench/Scanner.Analyzers/lang/README.md +++ b/src/Bench/StellaOps.Bench/Scanner.Analyzers/lang/README.md @@ -29,3 +29,5 @@ Results should be committed as deterministic CSV/JSON outputs with accompanying - Added two Python scenarios to `config.json`: the virtualenv sample (`python_site_packages_scan`) and the RECORD-heavy pip cache fixture (`python_pip_cache_fixture`). - Baseline run (Release build, 5 iterations) records means of **5.64 ms** (p95 18.29 ms) for the virtualenv and **5.86 ms** (p95 13.29 ms) for the pip cache verifier; raw numbers stored in `python/hash-throughput-20251023.csv`. - The pip cache fixture exercises `PythonRecordVerifier` with 12 RECORD rows (7 hashed) and mismatched layer coverage, giving a repeatable hash-validation throughput reference for regression gating. +- 2025-12-13: Added `python_layered_editable_fixture` scenario with `thresholdMs=2000` to guard container-root paths. +- 2025-12-13: Refreshed `baseline.csv` after Python analyzer discovery/VFS changes (see `src/Bench/StellaOps.Bench/Scanner.Analyzers/baseline.csv`). diff --git a/src/Bench/StellaOps.Bench/TASKS.md b/src/Bench/StellaOps.Bench/TASKS.md index b89cc97cf..b8f28e8ee 100644 --- a/src/Bench/StellaOps.Bench/TASKS.md +++ b/src/Bench/StellaOps.Bench/TASKS.md @@ -9,3 +9,5 @@ | BENCH-POLICY-20-002 | DONE (2025-12-11) | SPRINT_0512_0001_0001_bench | Policy delta benchmark (full vs delta) using baseline/delta NDJSON fixtures; outputs hashed. | `src/Bench/StellaOps.Bench/PolicyDelta` | | BENCH-SIG-26-001 | DONE (2025-12-11) | SPRINT_0512_0001_0001_bench | Reachability scoring harness with schema hash, 10k/50k fixtures, cache outputs for downstream benches. | `src/Bench/StellaOps.Bench/Signals` | | BENCH-SIG-26-002 | DONE (2025-12-11) | SPRINT_0512_0001_0001_bench | Policy evaluation cache bench (cold/warm/mixed) consuming reachability caches; outputs hashed. | `src/Bench/StellaOps.Bench/PolicyCache` | +| BENCH-SCANNER-ANALYZERS-405-008 | DONE (2025-12-13) | SPRINT_0405_0001_0001_scanner_python_detection_gaps.md | Extend Scanner analyzer microbench coverage for the Python analyzer (fixtures + thresholds + docs alignment). | `src/Bench/StellaOps.Bench/Scanner.Analyzers` | +| BENCH-SCANNER-ANALYZERS-407-009 | DONE (2025-12-13) | SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md | Add Bun analyzer scenario to microbench harness (config + baseline + wiring). | `src/Bench/StellaOps.Bench/Scanner.Analyzers` | diff --git a/src/Concelier/StellaOps.Concelier.WebService/Services/MessagingAdvisoryChunkCache.cs b/src/Concelier/StellaOps.Concelier.WebService/Services/MessagingAdvisoryChunkCache.cs new file mode 100644 index 000000000..a256a86e8 --- /dev/null +++ b/src/Concelier/StellaOps.Concelier.WebService/Services/MessagingAdvisoryChunkCache.cs @@ -0,0 +1,52 @@ +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; + +namespace StellaOps.Concelier.WebService.Services; + +/// +/// Advisory chunk cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +internal sealed class MessagingAdvisoryChunkCache : IAdvisoryChunkCache +{ + private readonly IDistributedCache _cache; + + public MessagingAdvisoryChunkCache(IDistributedCacheFactory cacheFactory) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = "advisory:chunk:", + }); + } + + public bool TryGet(in AdvisoryChunkCacheKey key, out AdvisoryChunkBuildResult result) + { + // Sync-over-async for compatibility with existing interface + // Consider migrating callers to async in future + var task = _cache.GetAsync(key.Value); + var cacheResult = task.AsTask().GetAwaiter().GetResult(); + + if (cacheResult.HasValue) + { + result = cacheResult.Value; + return true; + } + + result = null!; + return false; + } + + public void Set(in AdvisoryChunkCacheKey key, AdvisoryChunkBuildResult value, TimeSpan ttl) + { + if (ttl <= TimeSpan.Zero) + { + return; + } + + var entryOptions = new CacheEntryOptions { TimeToLive = ttl }; + // Sync-over-async for compatibility with existing interface + _cache.SetAsync(key.Value, value, entryOptions).AsTask().GetAwaiter().GetResult(); + } +} diff --git a/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj b/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj index eefa69635..2cccfaed1 100644 --- a/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj +++ b/src/Concelier/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj @@ -28,6 +28,7 @@ + diff --git a/src/Excititor/StellaOps.Excititor.WebService/Services/MessagingGraphOverlayCache.cs b/src/Excititor/StellaOps.Excititor.WebService/Services/MessagingGraphOverlayCache.cs new file mode 100644 index 000000000..cf04bd1a1 --- /dev/null +++ b/src/Excititor/StellaOps.Excititor.WebService/Services/MessagingGraphOverlayCache.cs @@ -0,0 +1,76 @@ +using Microsoft.Extensions.Options; +using StellaOps.Excititor.WebService.Contracts; +using StellaOps.Excititor.WebService.Options; +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; + +namespace StellaOps.Excititor.WebService.Services; + +/// +/// Graph overlay cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +internal sealed class MessagingGraphOverlayCache : IGraphOverlayCache +{ + private readonly IDistributedCache _cache; + private readonly IOptions _options; + private readonly TimeProvider _timeProvider; + + public MessagingGraphOverlayCache( + IDistributedCacheFactory cacheFactory, + IOptions options, + TimeProvider timeProvider) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = "graph-overlays:", + }); + } + + public async ValueTask TryGetAsync( + string tenant, + bool includeJustifications, + IReadOnlyList orderedPurls, + CancellationToken cancellationToken) + { + var key = BuildKey(tenant, includeJustifications, orderedPurls); + var result = await _cache.GetAsync(key, cancellationToken).ConfigureAwait(false); + + if (result.HasValue) + { + var cached = result.Value; + var ageMs = (long)Math.Max(0, (_timeProvider.GetUtcNow() - cached.CachedAt).TotalMilliseconds); + return new GraphOverlayCacheHit(cached.Items, ageMs); + } + + return null; + } + + public async ValueTask SaveAsync( + string tenant, + bool includeJustifications, + IReadOnlyList orderedPurls, + IReadOnlyList items, + DateTimeOffset cachedAt, + CancellationToken cancellationToken) + { + var key = BuildKey(tenant, includeJustifications, orderedPurls); + var ttl = TimeSpan.FromSeconds(Math.Max(1, _options.Value.OverlayTtlSeconds)); + var entry = new GraphOverlayCacheEntry(items.ToList(), cachedAt); + var entryOptions = new CacheEntryOptions { TimeToLive = ttl }; + + await _cache.SetAsync(key, entry, entryOptions, cancellationToken).ConfigureAwait(false); + } + + private static string BuildKey(string tenant, bool includeJustifications, IReadOnlyList orderedPurls) + => $"{tenant}:{includeJustifications}:{string.Join('|', orderedPurls)}"; +} + +/// +/// Cache entry for graph overlays. +/// +internal sealed record GraphOverlayCacheEntry(List Items, DateTimeOffset CachedAt); diff --git a/src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj b/src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj index 430b37832..61bf107d6 100644 --- a/src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj +++ b/src/Excititor/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj @@ -17,6 +17,7 @@ + diff --git a/src/Gateway/__Tests/StellaOps.Gateway.WebService.Tests/StellaOps.Gateway.WebService.Tests.csproj b/src/Gateway/__Tests/StellaOps.Gateway.WebService.Tests/StellaOps.Gateway.WebService.Tests.csproj index 2a50cea9f..a8dd90ffe 100644 --- a/src/Gateway/__Tests/StellaOps.Gateway.WebService.Tests/StellaOps.Gateway.WebService.Tests.csproj +++ b/src/Gateway/__Tests/StellaOps.Gateway.WebService.Tests/StellaOps.Gateway.WebService.Tests.csproj @@ -6,7 +6,7 @@ enable false false - + false diff --git a/src/Policy/StellaOps.Policy.Engine/Caching/MessagingPolicyEvaluationCache.cs b/src/Policy/StellaOps.Policy.Engine/Caching/MessagingPolicyEvaluationCache.cs new file mode 100644 index 000000000..7a2f5f517 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/Caching/MessagingPolicyEvaluationCache.cs @@ -0,0 +1,202 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; +using StellaOps.Policy.Engine.Options; + +namespace StellaOps.Policy.Engine.Caching; + +/// +/// Policy evaluation cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +public sealed class MessagingPolicyEvaluationCache : IPolicyEvaluationCache +{ + private readonly IDistributedCache _cache; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly TimeSpan _defaultTtl; + + private long _totalRequests; + private long _cacheHits; + private long _cacheMisses; + + public MessagingPolicyEvaluationCache( + IDistributedCacheFactory cacheFactory, + ILogger logger, + TimeProvider timeProvider, + IOptions options) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + + var cacheOptions = options?.Value.EvaluationCache ?? new PolicyEvaluationCacheOptions(); + _defaultTtl = TimeSpan.FromMinutes(cacheOptions.DefaultTtlMinutes); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = "pe:", + DefaultTtl = _defaultTtl, + }); + + _logger.LogInformation( + "Initialized MessagingPolicyEvaluationCache with provider {Provider}, TTL {Ttl}", + _cache.ProviderName, + _defaultTtl); + } + + public async Task GetAsync( + PolicyEvaluationCacheKey key, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _totalRequests); + + var cacheKey = key.ToCacheKey(); + var result = await _cache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false); + + if (result.HasValue) + { + var entry = result.Value; + var now = _timeProvider.GetUtcNow(); + + // Check if entry is still valid + if (entry.ExpiresAt > now) + { + Interlocked.Increment(ref _cacheHits); + return new PolicyEvaluationCacheResult(entry, true, MapSource(_cache.ProviderName)); + } + + // Entry expired - remove it + await _cache.InvalidateAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + + Interlocked.Increment(ref _cacheMisses); + return new PolicyEvaluationCacheResult(null, false, CacheSource.None); + } + + public async Task GetBatchAsync( + IReadOnlyList keys, + CancellationToken cancellationToken = default) + { + var found = new Dictionary(); + var notFound = new List(); + var hits = 0; + var misses = 0; + + foreach (var key in keys) + { + var result = await GetAsync(key, cancellationToken).ConfigureAwait(false); + if (result.Entry != null) + { + found[key] = result.Entry; + hits++; + } + else + { + notFound.Add(key); + misses++; + } + } + + var source = MapSource(_cache.ProviderName); + return new PolicyEvaluationCacheBatch + { + Found = found, + NotFound = notFound, + CacheHits = hits, + CacheMisses = misses, + InMemoryHits = source == CacheSource.InMemory ? hits : 0, + RedisHits = source == CacheSource.Redis ? hits : 0, + }; + } + + public async Task SetAsync( + PolicyEvaluationCacheKey key, + PolicyEvaluationCacheEntry entry, + CancellationToken cancellationToken = default) + { + var cacheKey = key.ToCacheKey(); + var now = _timeProvider.GetUtcNow(); + var expiresAt = entry.ExpiresAt > now ? entry.ExpiresAt : now.Add(_defaultTtl); + + var ttl = expiresAt - now; + if (ttl <= TimeSpan.Zero) + { + return; + } + + var options = new CacheEntryOptions { TimeToLive = ttl }; + await _cache.SetAsync(cacheKey, entry, options, cancellationToken).ConfigureAwait(false); + } + + public async Task SetBatchAsync( + IReadOnlyDictionary entries, + CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + + foreach (var (key, entry) in entries) + { + var cacheKey = key.ToCacheKey(); + var expiresAt = entry.ExpiresAt > now ? entry.ExpiresAt : now.Add(_defaultTtl); + + var ttl = expiresAt - now; + if (ttl <= TimeSpan.Zero) + { + continue; + } + + var options = new CacheEntryOptions { TimeToLive = ttl }; + await _cache.SetAsync(cacheKey, entry, options, cancellationToken).ConfigureAwait(false); + } + } + + public async Task InvalidateAsync( + PolicyEvaluationCacheKey key, + CancellationToken cancellationToken = default) + { + var cacheKey = key.ToCacheKey(); + await _cache.InvalidateAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + + public async Task InvalidateByPolicyDigestAsync( + string policyDigest, + CancellationToken cancellationToken = default) + { + // Pattern: pe::* + var pattern = $"{policyDigest}:*"; + var count = await _cache.InvalidateByPatternAsync(pattern, cancellationToken).ConfigureAwait(false); + + _logger.LogDebug( + "Invalidated {Count} cache entries for policy digest {Digest}", + count, + policyDigest); + } + + public PolicyEvaluationCacheStats GetStats() + { + var source = MapSource(_cache.ProviderName); + var hits = Interlocked.Read(ref _cacheHits); + + return new PolicyEvaluationCacheStats + { + TotalRequests = Interlocked.Read(ref _totalRequests), + CacheHits = hits, + CacheMisses = Interlocked.Read(ref _cacheMisses), + InMemoryHits = source == CacheSource.InMemory ? hits : 0, + RedisHits = source == CacheSource.Redis ? hits : 0, + RedisFallbacks = 0, + ItemCount = 0, // Not available from IDistributedCache + EvictionCount = 0, // Not available from IDistributedCache + }; + } + + private static CacheSource MapSource(string providerName) => providerName.ToLowerInvariant() switch + { + "inmemory" => CacheSource.InMemory, + "valkey" => CacheSource.Redis, + "redis" => CacheSource.Redis, + _ => CacheSource.None, + }; +} diff --git a/src/Policy/StellaOps.Policy.Engine/DependencyInjection/PolicyEngineServiceCollectionExtensions.cs b/src/Policy/StellaOps.Policy.Engine/DependencyInjection/PolicyEngineServiceCollectionExtensions.cs index 7df817ca8..39c3e9fb0 100644 --- a/src/Policy/StellaOps.Policy.Engine/DependencyInjection/PolicyEngineServiceCollectionExtensions.cs +++ b/src/Policy/StellaOps.Policy.Engine/DependencyInjection/PolicyEngineServiceCollectionExtensions.cs @@ -29,8 +29,8 @@ public static class PolicyEngineServiceCollectionExtensions // Core compilation and evaluation services services.TryAddSingleton(); - // Cache - services.TryAddSingleton(); + // Cache - uses IDistributedCacheFactory for transport flexibility + services.TryAddSingleton(); // Runtime evaluation services.TryAddSingleton(); diff --git a/src/Policy/StellaOps.Policy.Engine/Gates/PolicyGateEvaluator.cs b/src/Policy/StellaOps.Policy.Engine/Gates/PolicyGateEvaluator.cs index 7578becae..341acefb9 100644 --- a/src/Policy/StellaOps.Policy.Engine/Gates/PolicyGateEvaluator.cs +++ b/src/Policy/StellaOps.Policy.Engine/Gates/PolicyGateEvaluator.cs @@ -470,7 +470,7 @@ public sealed class PolicyGateEvaluator : IPolicyGateEvaluator Name = "LatticeState", Result = PolicyGateResultType.Warn, Reason = $"{latticeState} may indicate false positive for affected", - Note = "Consider review: evidence suggests code may not be reachable" + Note = "Consider review: evidence suggests code may not be reachable (possible false positive)" }; default: diff --git a/src/Policy/StellaOps.Policy.Engine/ReachabilityFacts/MessagingReachabilityFactsOverlayCache.cs b/src/Policy/StellaOps.Policy.Engine/ReachabilityFacts/MessagingReachabilityFactsOverlayCache.cs new file mode 100644 index 000000000..27036deb8 --- /dev/null +++ b/src/Policy/StellaOps.Policy.Engine/ReachabilityFacts/MessagingReachabilityFactsOverlayCache.cs @@ -0,0 +1,180 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Messaging; +using StellaOps.Messaging.Abstractions; +using StellaOps.Policy.Engine.Options; + +namespace StellaOps.Policy.Engine.ReachabilityFacts; + +/// +/// Reachability facts overlay cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +public sealed class MessagingReachabilityFactsOverlayCache : IReachabilityFactsOverlayCache +{ + private readonly IDistributedCache _cache; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly TimeSpan _defaultTtl; + + private long _totalRequests; + private long _cacheHits; + private long _cacheMisses; + + public MessagingReachabilityFactsOverlayCache( + IDistributedCacheFactory cacheFactory, + ILogger logger, + TimeProvider timeProvider, + IOptions options) + { + ArgumentNullException.ThrowIfNull(cacheFactory); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + + var cacheOptions = options?.Value.ReachabilityCache ?? new ReachabilityFactsCacheOptions(); + _defaultTtl = TimeSpan.FromMinutes(cacheOptions.DefaultTtlMinutes); + + _cache = cacheFactory.Create(new CacheOptions + { + KeyPrefix = "rf:", + DefaultTtl = _defaultTtl, + }); + + _logger.LogInformation( + "Initialized MessagingReachabilityFactsOverlayCache with provider {Provider}, TTL {Ttl}", + _cache.ProviderName, + _defaultTtl); + } + + public async Task<(ReachabilityFact? Fact, bool CacheHit)> GetAsync( + ReachabilityFactKey key, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref _totalRequests); + + var cacheKey = key.ToCacheKey(); + var result = await _cache.GetAsync(cacheKey, cancellationToken).ConfigureAwait(false); + + if (result.HasValue) + { + var fact = result.Value; + var now = _timeProvider.GetUtcNow(); + + // Check if entry is still valid + if (!fact.ExpiresAt.HasValue || fact.ExpiresAt.Value > now) + { + Interlocked.Increment(ref _cacheHits); + return (fact, true); + } + + // Entry expired - remove it + await _cache.InvalidateAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + + Interlocked.Increment(ref _cacheMisses); + return (null, false); + } + + public async Task GetBatchAsync( + IReadOnlyList keys, + CancellationToken cancellationToken = default) + { + var found = new Dictionary(); + var notFound = new List(); + var cacheHits = 0; + var cacheMisses = 0; + + foreach (var key in keys) + { + var (fact, hit) = await GetAsync(key, cancellationToken).ConfigureAwait(false); + if (fact != null) + { + found[key] = fact; + cacheHits++; + } + else + { + notFound.Add(key); + cacheMisses++; + } + } + + return new ReachabilityFactsBatch + { + Found = found, + NotFound = notFound, + CacheHits = cacheHits, + CacheMisses = cacheMisses, + }; + } + + public async Task SetAsync( + ReachabilityFactKey key, + ReachabilityFact fact, + CancellationToken cancellationToken = default) + { + var cacheKey = key.ToCacheKey(); + var now = _timeProvider.GetUtcNow(); + var ttl = fact.ExpiresAt.HasValue && fact.ExpiresAt.Value > now + ? fact.ExpiresAt.Value - now + : _defaultTtl; + + if (ttl <= TimeSpan.Zero) + { + return; + } + + var options = new CacheEntryOptions { TimeToLive = ttl }; + await _cache.SetAsync(cacheKey, fact, options, cancellationToken).ConfigureAwait(false); + } + + public async Task SetBatchAsync( + IReadOnlyDictionary facts, + CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + + foreach (var (key, fact) in facts) + { + var cacheKey = key.ToCacheKey(); + var ttl = fact.ExpiresAt.HasValue && fact.ExpiresAt.Value > now + ? fact.ExpiresAt.Value - now + : _defaultTtl; + + if (ttl <= TimeSpan.Zero) + { + continue; + } + + var options = new CacheEntryOptions { TimeToLive = ttl }; + await _cache.SetAsync(cacheKey, fact, options, cancellationToken).ConfigureAwait(false); + } + } + + public async Task InvalidateAsync(ReachabilityFactKey key, CancellationToken cancellationToken = default) + { + var cacheKey = key.ToCacheKey(); + await _cache.InvalidateAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + + public async Task InvalidateTenantAsync(string tenantId, CancellationToken cancellationToken = default) + { + // Pattern: rf::* + var pattern = $"{tenantId}:*"; + var count = await _cache.InvalidateByPatternAsync(pattern, cancellationToken).ConfigureAwait(false); + + _logger.LogDebug("Invalidated {Count} cache entries for tenant {TenantId}", count, tenantId); + } + + public ReachabilityFactsCacheStats GetStats() + { + return new ReachabilityFactsCacheStats + { + TotalRequests = Interlocked.Read(ref _totalRequests), + CacheHits = Interlocked.Read(ref _cacheHits), + CacheMisses = Interlocked.Read(ref _cacheMisses), + ItemCount = 0, // Not available from IDistributedCache + EvictionCount = 0, // Not available from IDistributedCache + }; + } +} diff --git a/src/Policy/StellaOps.Policy.Engine/TASKS.md b/src/Policy/StellaOps.Policy.Engine/TASKS.md index 21bd4592a..85c9afb43 100644 --- a/src/Policy/StellaOps.Policy.Engine/TASKS.md +++ b/src/Policy/StellaOps.Policy.Engine/TASKS.md @@ -4,4 +4,4 @@ This file mirrors sprint work for the Policy Engine module. | Task ID | Sprint | Status | Notes | | --- | --- | --- | --- | -| `POLICY-GATE-401-033` | `docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md` | DOING | Gate `unreachable` reachability facts: missing evidence ref or low confidence => `under_investigation`; add tests and docs. | +| `POLICY-GATE-401-033` | `docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md` | DONE (2025-12-13) | Implemented PolicyGateEvaluator (lattice/uncertainty/evidence completeness) and aligned tests/docs; see `src/Policy/StellaOps.Policy.Engine/Gates/PolicyGateEvaluator.cs` and `src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Gates/PolicyGateEvaluatorTests.cs`. | diff --git a/src/Policy/StellaOps.Policy.Engine/Tenancy/TenantContextMiddleware.cs b/src/Policy/StellaOps.Policy.Engine/Tenancy/TenantContextMiddleware.cs index a62587712..a542a39bc 100644 --- a/src/Policy/StellaOps.Policy.Engine/Tenancy/TenantContextMiddleware.cs +++ b/src/Policy/StellaOps.Policy.Engine/Tenancy/TenantContextMiddleware.cs @@ -55,13 +55,20 @@ public sealed partial class TenantContextMiddleware // Set tenant context for the request tenantContextAccessor.TenantContext = validationResult.Context; - using (_logger.BeginScope(new Dictionary + try { - ["tenant_id"] = validationResult.Context?.TenantId, - ["project_id"] = validationResult.Context?.ProjectId - })) + using (_logger.BeginScope(new Dictionary + { + ["tenant_id"] = validationResult.Context?.TenantId, + ["project_id"] = validationResult.Context?.ProjectId + })) + { + await _next(context); + } + } + finally { - await _next(context); + tenantContextAccessor.TenantContext = null; } } diff --git a/src/Policy/__Libraries/StellaOps.Policy.Storage.Postgres/Migration/PolicyMigrator.cs b/src/Policy/__Libraries/StellaOps.Policy.Storage.Postgres/Migration/PolicyMigrator.cs index e70266b05..00a60c854 100644 --- a/src/Policy/__Libraries/StellaOps.Policy.Storage.Postgres/Migration/PolicyMigrator.cs +++ b/src/Policy/__Libraries/StellaOps.Policy.Storage.Postgres/Migration/PolicyMigrator.cs @@ -5,11 +5,11 @@ using StellaOps.Policy.Storage.Postgres.Repositories; namespace StellaOps.Policy.Storage.Postgres.Migration; /// -/// Handles migration of policy data from MongoDB to PostgreSQL. +/// Handles migration of policy data from legacy storage to PostgreSQL. /// Task references: PG-T4.9, PG-T4.10, PG-T4.11 /// /// -/// This migrator converts policy packs and their versions from MongoDB documents +/// This migrator converts policy packs and their versions from legacy storage documents /// to PostgreSQL entities while preserving version history and active version settings. /// public sealed class PolicyMigrator @@ -207,10 +207,10 @@ public sealed class PolicyMigrator } /// - /// Verifies that migrated data matches between MongoDB and PostgreSQL. + /// Verifies that migrated data matches expected counts in PostgreSQL. /// /// Tenant to verify. - /// Expected pack count from MongoDB. + /// Expected pack count from source data. /// Expected version counts per pack. /// Cancellation token. /// Verification result. @@ -314,7 +314,7 @@ public sealed class PolicyMigrator /// public sealed class PackMigrationData { - /// Source system identifier (MongoDB _id). + /// Source system identifier. public required string SourceId { get; init; } /// Tenant identifier. diff --git a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Tenancy/TenantContextTests.cs b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Tenancy/TenantContextTests.cs index 427338fc9..5f75a372d 100644 --- a/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Tenancy/TenantContextTests.cs +++ b/src/Policy/__Tests/StellaOps.Policy.Engine.Tests/Tenancy/TenantContextTests.cs @@ -41,7 +41,7 @@ public sealed class TenantContextTests public void TenantContext_ForTenant_ThrowsOnNullTenantId() { // Act & Assert - Assert.Throws(() => TenantContext.ForTenant(null!)); + Assert.Throws(() => TenantContext.ForTenant(null!)); } [Fact] @@ -156,9 +156,15 @@ public sealed class TenantContextMiddlewareTests public async Task Middleware_WithValidTenantHeader_SetsTenantContext() { // Arrange + TenantContext? capturedContext = null; var nextCalled = false; var middleware = new TenantContextMiddleware( - _ => { nextCalled = true; return Task.CompletedTask; }, + _ => + { + nextCalled = true; + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -169,16 +175,22 @@ public sealed class TenantContextMiddlewareTests // Assert Assert.True(nextCalled); - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal("tenant-123", _tenantAccessor.TenantContext.TenantId); + Assert.NotNull(capturedContext); + Assert.Equal("tenant-123", capturedContext!.TenantId); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] public async Task Middleware_WithTenantAndProjectHeaders_SetsBothInContext() { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -188,9 +200,10 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal("tenant-123", _tenantAccessor.TenantContext.TenantId); - Assert.Equal("project-456", _tenantAccessor.TenantContext.ProjectId); + Assert.NotNull(capturedContext); + Assert.Equal("tenant-123", capturedContext!.TenantId); + Assert.Equal("project-456", capturedContext.ProjectId); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] @@ -218,6 +231,7 @@ public sealed class TenantContextMiddlewareTests public async Task Middleware_MissingTenantHeaderNotRequired_UsesDefaultTenant() { // Arrange + TenantContext? capturedContext = null; var optionsNotRequired = new TenantContextOptions { Enabled = true, @@ -225,7 +239,11 @@ public sealed class TenantContextMiddlewareTests }; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(optionsNotRequired), _logger); @@ -235,8 +253,9 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal(TenantContextConstants.DefaultTenantId, _tenantAccessor.TenantContext.TenantId); + Assert.NotNull(capturedContext); + Assert.Equal(TenantContextConstants.DefaultTenantId, capturedContext!.TenantId); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] @@ -286,8 +305,13 @@ public sealed class TenantContextMiddlewareTests public async Task Middleware_ValidTenantIdFormat_Passes(string tenantId) { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -297,8 +321,9 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal(tenantId, _tenantAccessor.TenantContext.TenantId); + Assert.NotNull(capturedContext); + Assert.Equal(tenantId, capturedContext!.TenantId); + Assert.Null(_tenantAccessor.TenantContext); } [Theory] @@ -351,8 +376,13 @@ public sealed class TenantContextMiddlewareTests public async Task Middleware_ValidProjectIdFormat_Passes(string projectId) { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -362,16 +392,22 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal(projectId, _tenantAccessor.TenantContext.ProjectId); + Assert.NotNull(capturedContext); + Assert.Equal(projectId, capturedContext!.ProjectId); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] public async Task Middleware_WithWriteScope_SetsCanWriteTrue() { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -387,16 +423,22 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.True(_tenantAccessor.TenantContext.CanWrite); + Assert.NotNull(capturedContext); + Assert.True(capturedContext!.CanWrite); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] public async Task Middleware_WithoutWriteScope_SetsCanWriteFalse() { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -412,16 +454,22 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.False(_tenantAccessor.TenantContext.CanWrite); + Assert.NotNull(capturedContext); + Assert.False(capturedContext!.CanWrite); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] public async Task Middleware_ExtractsActorIdFromSubClaim() { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -433,16 +481,22 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal("user-id-123", _tenantAccessor.TenantContext.ActorId); + Assert.NotNull(capturedContext); + Assert.Equal("user-id-123", capturedContext!.ActorId); + Assert.Null(_tenantAccessor.TenantContext); } [Fact] public async Task Middleware_ExtractsActorIdFromHeader() { // Arrange + TenantContext? capturedContext = null; var middleware = new TenantContextMiddleware( - _ => Task.CompletedTask, + _ => + { + capturedContext = _tenantAccessor.TenantContext; + return Task.CompletedTask; + }, MsOptions.Options.Create(_options), _logger); @@ -453,8 +507,9 @@ public sealed class TenantContextMiddlewareTests await middleware.InvokeAsync(context, _tenantAccessor); // Assert - Assert.NotNull(_tenantAccessor.TenantContext); - Assert.Equal("service-account-123", _tenantAccessor.TenantContext.ActorId); + Assert.NotNull(capturedContext); + Assert.Equal("service-account-123", capturedContext!.ActorId); + Assert.Null(_tenantAccessor.TenantContext); } private static DefaultHttpContext CreateHttpContext( diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md new file mode 100644 index 000000000..a2c16c594 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/AGENTS.md @@ -0,0 +1,45 @@ +# StellaOps.Scanner.Analyzers.Lang.Bun - Agent Charter + +## Role +Deliver the Bun analyzer plug-in that inventories npm-ecosystem dependencies from Bun-managed projects and emits deterministic, evidence-backed component records for Scanner Workers. + +## Scope +- Bun project discovery (including common container layer layouts like `layers/*`, `.layers/*`, and `layer*`). +- Parse `bun.lock` (text lockfile, v1) and reconcile scopes (prod/dev/optional/peer) deterministically without executing Bun. +- Installed inventory from `node_modules/**/package.json` and Bun's isolated linker store (`node_modules/.bun/**/package.json`), with symlink safety. +- Declared-only fallback from `package.json` when no install/lock evidence exists, using safe identities (no invalid range-as-version PURLs). +- Patched dependency attribution (`patchedDependencies`, `patches/`, `.patches/`) with version-specific mapping and no absolute path leakage. +- Plug-in manifest/DI bootstrap maintenance for Worker loading. + +## Out of Scope +- Parsing `bun.lockb` (binary lockfile) unless explicitly scheduled. +- Running `bun` or fetching registries (offline-first always). +- Vulnerability correlation, policy evaluation, or UI/export formatting. + +## Expectations +- Offline-first: no process execution, no network calls, no reliance on host-global caches. +- Determinism: stable ordering, explicit bounds, normalized path separators, and deterministic "skipped" markers when limits are hit. +- Identity safety: never emit `pkg:npm/...@`; use `AddFromExplicitKey` for non-concrete versions and for non-registry sources. +- Evidence: locators are project-relative (no drive letters or host roots) and golden-tested; file hashing is size-bounded. +- Container bounds: discovery must be bounded and must never recurse into `node_modules/`. + +## Dependencies +- Shared analyzer infrastructure: `src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang`. +- Node analyzer conventions where contracts overlap (container roots, evidence/locator patterns). +- Scanner Surface filesystem normalization helpers (via `LanguageAnalyzerContext`). + +## Testing & Artifacts +- Tests: `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests`. +- Fixtures: `src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/**` (deterministic inputs and golden outputs). +- Optional benchmarks (only if perf risks materialize): `src/Bench/StellaOps.Bench/Scanner.Analyzers`. + +## Required Reading +- `docs/modules/scanner/architecture.md` +- `docs/modules/scanner/prep/bun-analyzer-design.md` +- `docs/modules/scanner/bun-analyzer-gotchas.md` +- `docs/implplan/SPRINT_0407_0001_0001_scanner_bun_detection_gaps.md` + +## Working Agreement +1. Update task status to `DOING`/`DONE` in both `/docs/implplan/SPRINT_*.md` and the local `TASKS.md` as work progresses. +2. Every behavior change is covered by fixtures + golden tests (and/or unit tests for parsers). +3. If a contract decision is required (identity, evidence locators, container layout), mark the affected task `BLOCKED` in the sprint and record the exact decision needed under **Decisions & Risks**. diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/BunLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/BunLanguageAnalyzer.cs index 91f3fabb7..c6ac3d4ca 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/BunLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/BunLanguageAnalyzer.cs @@ -38,6 +38,13 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer continue; } + // Declared-only fallback for bun markers (package.json + bunfig.toml) when no lock/install evidence exists. + if (classification.Kind == BunInputKind.None) + { + EmitDeclaredOnlyDependencies(writer, context, projectRoot, cancellationToken); + continue; + } + // Parse workspace info for direct dependency detection var workspaceInfo = BunWorkspaceHelper.ParseWorkspaceInfo(projectRoot); @@ -45,13 +52,37 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer var bunConfig = BunConfigHelper.ParseConfig(projectRoot); // Stage 3: Collect packages based on classification + string? lockfileRelativePath = null; + string? lockfileSha256 = null; + string? lockfileHashSkipReason = null; + if (classification.HasTextLockfile && !string.IsNullOrWhiteSpace(classification.TextLockfilePath)) + { + lockfileRelativePath = context.GetRelativePath(classification.TextLockfilePath!); + if (string.IsNullOrWhiteSpace(lockfileRelativePath)) + { + lockfileRelativePath = "bun.lock"; + } + + lockfileSha256 = BunEvidenceHasher.TryComputeBoundedSha256( + classification.TextLockfilePath!, + BunEvidenceHasher.MaxLockfileEvidenceBytes, + out lockfileHashSkipReason); + } + IReadOnlyList packages; if (classification.Kind == BunInputKind.InstalledModules) { // Prefer installed modules when available - var lockData = classification.HasTextLockfile - ? await BunLockParser.ParseAsync(classification.TextLockfilePath!, cancellationToken).ConfigureAwait(false) - : null; + BunLockData? lockData = null; + if (classification.HasTextLockfile) + { + lockData = await BunLockParser.ParseAsync(classification.TextLockfilePath!, cancellationToken).ConfigureAwait(false); + if (!lockData.AllEntries.IsEmpty) + { + var declared = BunDeclaredDependencyCollector.Collect(projectRoot); + lockData = BunLockScopeClassifier.Classify(lockData, declared); + } + } packages = BunInstalledCollector.Collect(context, projectRoot, lockData, cancellationToken); } @@ -59,6 +90,12 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer { // Fall back to lockfile parsing var lockData = await BunLockParser.ParseAsync(classification.TextLockfilePath!, cancellationToken).ConfigureAwait(false); + if (!lockData.AllEntries.IsEmpty) + { + var declared = BunDeclaredDependencyCollector.Collect(projectRoot); + lockData = BunLockScopeClassifier.Classify(lockData, declared); + } + packages = BunLockInventory.ExtractPackages(lockData, classification.IncludeDev); } else @@ -70,13 +107,13 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer // Mark direct, patched dependencies and custom registries foreach (var package in packages) { + package.LockfilePath = lockfileRelativePath; + package.LockfileSha256 = lockfileSha256; + package.LockfileHashSkipReason = lockfileHashSkipReason; + package.IsDirect = workspaceInfo.DirectDependencies.ContainsKey(package.Name); - if (workspaceInfo.PatchedDependencies.TryGetValue(package.Name, out var patchFile)) - { - package.IsPatched = true; - package.PatchFile = patchFile; - } + ApplyPatchMetadata(workspaceInfo, package); // Check for custom registry (scoped or default) if (bunConfig.HasCustomRegistry) @@ -98,26 +135,176 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer // Stage 4: Normalize and emit var normalized = BunPackageNormalizer.Normalize(packages); - foreach (var package in normalized.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal)) + + var emissionPlans = normalized + .Select(package => + { + if (BunVersionSpec.IsConcreteNpmVersion(package.Version)) + { + return new EmissionPlan(package, package.ComponentKey, UsePurl: true, ComponentKey: null); + } + + var originLocator = BuildNonConcreteOriginLocator(package); + var versionSpec = package.Version; + var componentKey = LanguageExplicitKey.Create(Id, "npm", package.Name, versionSpec, originLocator); + return new EmissionPlan(package, componentKey, UsePurl: false, ComponentKey: componentKey); + }) + .OrderBy(static p => p.SortKey, StringComparer.Ordinal); + + foreach (var plan in emissionPlans) { cancellationToken.ThrowIfCancellationRequested(); - var metadata = package.CreateMetadata(); + var package = plan.Package; var evidence = package.CreateEvidence(); - writer.AddFromPurl( - analyzerId: Id, - purl: package.Purl, - name: package.Name, - version: package.Version, - type: "npm", - metadata: metadata, - evidence: evidence, - usedByEntrypoint: false); + if (plan.UsePurl) + { + var metadata = package.CreateMetadata(); + writer.AddFromPurl( + analyzerId: Id, + purl: package.Purl, + name: package.Name, + version: package.Version, + type: "npm", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: false); + } + else + { + var metadata = new SortedDictionary(StringComparer.Ordinal); + foreach (var entry in package.CreateMetadata()) + { + metadata[entry.Key] = entry.Value; + } + + metadata["nonConcreteVersion"] = "true"; + metadata["versionSpec"] = package.Version; + + writer.AddFromExplicitKey( + analyzerId: Id, + componentKey: plan.ComponentKey!, + purl: null, + name: package.Name, + version: null, + type: "npm", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: false); + } } } } + private static void ApplyPatchMetadata(BunWorkspaceHelper.WorkspaceInfo workspaceInfo, BunPackage package) + { + if (workspaceInfo.PatchedDependencies.Count == 0) + { + return; + } + + var versionKey = $"{package.Name}@{package.Version}"; + if (workspaceInfo.PatchedDependencies.TryGetValue(versionKey, out var patchFile)) + { + package.IsPatched = true; + package.PatchFile = patchFile; + return; + } + + if (workspaceInfo.PatchedDependencies.TryGetValue(package.Name, out patchFile) && + IsNameOnlyPatchUnambiguous(workspaceInfo.PatchedDependencies, package.Name)) + { + package.IsPatched = true; + package.PatchFile = patchFile; + } + } + + private static bool IsNameOnlyPatchUnambiguous(IReadOnlyDictionary patchedDependencies, string packageName) + { + var prefix = $"{packageName}@"; + foreach (var key in patchedDependencies.Keys) + { + if (key.StartsWith(prefix, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private void EmitDeclaredOnlyDependencies( + LanguageComponentWriter writer, + LanguageAnalyzerContext context, + string projectRoot, + CancellationToken cancellationToken) + { + var declared = BunDeclaredDependencyCollector.Collect(projectRoot); + if (declared.Count == 0) + { + return; + } + + var packageJsonPath = Path.Combine(projectRoot, "package.json"); + var relativePackageJson = context.GetRelativePath(packageJsonPath); + if (string.IsNullOrWhiteSpace(relativePackageJson)) + { + relativePackageJson = "package.json"; + } + + var packageJsonHash = BunEvidenceHasher.TryComputeBoundedSha256( + packageJsonPath, + BunEvidenceHasher.MaxPackageJsonEvidenceBytes, + out var hashSkipReason); + + foreach (var dep in declared) + { + cancellationToken.ThrowIfCancellationRequested(); + + var locator = $"{relativePackageJson}#{dep.Section}"; + var componentKey = LanguageExplicitKey.Create(Id, "npm", dep.Name, dep.VersionSpec, locator); + + var metadata = new List>(12) + { + new("declaredOnly", "true"), + new("declared.source", "package.json"), + new("declared.locator", locator), + new("declared.versionSpec", dep.VersionSpec), + new("declared.scope", dep.Scope), + new("declared.sourceType", ClassifyDeclaredSourceType(dep.VersionSpec)), + new("packageManager", "bun"), + }; + + if (!string.IsNullOrEmpty(hashSkipReason)) + { + metadata.Add(new KeyValuePair("packageJson.hashSkipped", "true")); + metadata.Add(new KeyValuePair("packageJson.hashSkipReason", hashSkipReason)); + } + + var evidence = new[] + { + new LanguageComponentEvidence( + LanguageEvidenceKind.File, + "package.json", + relativePackageJson, + null, + packageJsonHash) + }; + + writer.AddFromExplicitKey( + analyzerId: Id, + componentKey: componentKey, + purl: null, + name: dep.Name, + version: null, + type: "npm", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: false); + } + } + private void EmitBinaryLockfileRemediation(LanguageComponentWriter writer, LanguageAnalyzerContext context, string projectRoot) { var relativePath = context.GetRelativePath(projectRoot); @@ -149,4 +336,73 @@ public sealed class BunLanguageAnalyzer : ILanguageAnalyzer metadata: metadata, evidence: evidence); } + + private static string ClassifyDeclaredSourceType(string spec) + { + if (string.IsNullOrWhiteSpace(spec)) + { + return "unknown"; + } + + var value = spec.Trim(); + + if (value.StartsWith("workspace:", StringComparison.OrdinalIgnoreCase)) + { + return "workspace"; + } + + if (value.StartsWith("link:", StringComparison.OrdinalIgnoreCase)) + { + return "link"; + } + + if (value.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + { + return "file"; + } + + if (value.StartsWith("git+", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("git://", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("github:", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("gitlab:", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("bitbucket:", StringComparison.OrdinalIgnoreCase)) + { + return "git"; + } + + if (value.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + return "tarball"; + } + + if (value is "latest" or "next" or "beta" or "alpha" or "canary") + { + return "tag"; + } + + if (value.Length > 0 && value[0] == '.' || + value.Length > 0 && value[0] == '/' || + value.Contains('\\')) + { + return "path"; + } + + return "range"; + } + + private static string BuildNonConcreteOriginLocator(BunPackage package) + { + if (!string.IsNullOrWhiteSpace(package.LogicalPath)) + { + return NormalizePath(Path.Combine(package.LogicalPath!, "package.json")); + } + + var lockfilePath = string.IsNullOrWhiteSpace(package.LockfilePath) ? "bun.lock" : package.LockfilePath!; + return $"{NormalizePath(lockfilePath)}:packages[{package.Name}@{package.Version}]"; + } + + private static string NormalizePath(string path) => path.Replace('\\', '/'); + + private sealed record EmissionPlan(BunPackage Package, string SortKey, bool UsePurl, string? ComponentKey); } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunDeclaredDependencyCollector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunDeclaredDependencyCollector.cs new file mode 100644 index 000000000..1226d71ab --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunDeclaredDependencyCollector.cs @@ -0,0 +1,80 @@ +using System.Collections.Immutable; +using System.Text.Json; + +namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal; + +internal static class BunDeclaredDependencyCollector +{ + internal sealed record DeclaredDependency( + string Name, + string VersionSpec, + string Section, + string Scope); + + public static IReadOnlyList Collect(string projectRoot) + { + ArgumentException.ThrowIfNullOrWhiteSpace(projectRoot); + + var packageJsonPath = Path.Combine(projectRoot, "package.json"); + if (!File.Exists(packageJsonPath)) + { + return ImmutableArray.Empty; + } + + try + { + var content = File.ReadAllText(packageJsonPath); + using var document = JsonDocument.Parse(content); + var root = document.RootElement; + + var results = new List(); + + AddDependencies(results, root, "dependencies", "prod"); + AddDependencies(results, root, "devDependencies", "dev"); + AddDependencies(results, root, "optionalDependencies", "optional"); + AddDependencies(results, root, "peerDependencies", "peer"); + + results.Sort(static (left, right) => + { + var nameCompare = string.CompareOrdinal(left.Name, right.Name); + if (nameCompare != 0) + { + return nameCompare; + } + + return string.CompareOrdinal(left.Scope, right.Scope); + }); + + return results.ToImmutableArray(); + } + catch (JsonException) + { + return ImmutableArray.Empty; + } + catch (IOException) + { + return ImmutableArray.Empty; + } + } + + private static void AddDependencies(List results, JsonElement root, string section, string scope) + { + if (!root.TryGetProperty(section, out var deps) || deps.ValueKind != JsonValueKind.Object) + { + return; + } + + foreach (var dep in deps.EnumerateObject()) + { + var name = dep.Name; + var versionSpec = dep.Value.ValueKind == JsonValueKind.String ? dep.Value.GetString() : null; + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(versionSpec)) + { + continue; + } + + results.Add(new DeclaredDependency(name.Trim(), versionSpec!.Trim(), section, scope)); + } + } +} + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunEvidenceHasher.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunEvidenceHasher.cs new file mode 100644 index 000000000..dd63d3bb3 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunEvidenceHasher.cs @@ -0,0 +1,45 @@ +using System.Security.Cryptography; + +namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal; + +internal static class BunEvidenceHasher +{ + internal const int MaxPackageJsonEvidenceBytes = 1024 * 1024; // 1 MiB + internal const int MaxLockfileEvidenceBytes = 50 * 1024 * 1024; // 50 MiB (matches BunLockParser cap) + + internal static string? TryComputeBoundedSha256(string path, int maxBytes, out string? skipReason) + { + skipReason = null; + + try + { + var info = new FileInfo(path); + if (!info.Exists) + { + skipReason = "missing"; + return null; + } + + if (info.Length > maxBytes) + { + skipReason = $"size>{maxBytes}"; + return null; + } + + using var stream = File.OpenRead(path); + var hash = SHA256.HashData(stream); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + catch (UnauthorizedAccessException) + { + skipReason = "unauthorized"; + return null; + } + catch (IOException) + { + skipReason = "io"; + return null; + } + } +} + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunInstalledCollector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunInstalledCollector.cs index 3f466d3bf..258b758f3 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunInstalledCollector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunInstalledCollector.cs @@ -1,4 +1,5 @@ using System.Collections.Immutable; +using System.Security.Cryptography; using System.Text.Json; namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal; @@ -186,9 +187,30 @@ internal static class BunInstalledCollector { try { - var content = File.ReadAllText(packageJsonPath); - using var document = JsonDocument.Parse(content); - var root = document.RootElement; + var fileInfo = new FileInfo(packageJsonPath); + string? packageJsonSha256 = null; + string? packageJsonHashSkipReason = null; + + JsonElement root; + if (fileInfo.Exists && fileInfo.Length <= BunEvidenceHasher.MaxPackageJsonEvidenceBytes) + { + var bytes = File.ReadAllBytes(packageJsonPath); + packageJsonSha256 = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + + using var document = JsonDocument.Parse(bytes); + root = document.RootElement.Clone(); + } + else + { + if (fileInfo.Exists && fileInfo.Length > BunEvidenceHasher.MaxPackageJsonEvidenceBytes) + { + packageJsonHashSkipReason = $"size>{BunEvidenceHasher.MaxPackageJsonEvidenceBytes}"; + } + + using var stream = File.OpenRead(packageJsonPath); + using var document = JsonDocument.Parse(stream); + root = document.RootElement.Clone(); + } if (!root.TryGetProperty("name", out var nameElement)) { @@ -221,12 +243,18 @@ internal static class BunInstalledCollector relativePath, relativeRealPath, isPrivate, - lockEntry); + lockEntry, + packageJsonSha256: packageJsonSha256, + packageJsonHashSkipReason: packageJsonHashSkipReason); } catch (JsonException) { return null; } + catch (UnauthorizedAccessException) + { + return null; + } catch (IOException) { return null; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockEntry.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockEntry.cs index f5d36b03d..8b5df2a1b 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockEntry.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockEntry.cs @@ -12,6 +12,7 @@ internal sealed class BunLockEntry public bool IsDev { get; init; } public bool IsOptional { get; init; } public bool IsPeer { get; init; } + public bool ScopeUnknown { get; init; } /// /// Source type: npm, git, tarball, file, link, workspace. @@ -31,5 +32,7 @@ internal sealed class BunLockEntry /// /// Dependencies of this package (for transitive analysis). /// - public IReadOnlyList Dependencies { get; init; } = Array.Empty(); + public IReadOnlyList Dependencies { get; init; } = Array.Empty(); } + +internal sealed record BunLockDependency(string Name, string? Specifier, bool IsOptionalPeer); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockParser.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockParser.cs index bd03a2d4e..68bc8e667 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockParser.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockParser.cs @@ -141,13 +141,24 @@ internal static class BunLockParser var resolved = element[0].GetString(); var integrity = element.GetArrayLength() > 1 ? element[1].GetString() : null; - // Parse dependencies from element[2] if present - var dependencies = new List(); + // Parse dependencies from element[2] if present. + var dependencies = new List(); if (element.GetArrayLength() > 2 && element[2].ValueKind == JsonValueKind.Object) { foreach (var dep in element[2].EnumerateObject()) { - dependencies.Add(dep.Name); + var depSpecifier = dep.Value.ValueKind == JsonValueKind.String ? dep.Value.GetString() : null; + dependencies.Add(new BunLockDependency(dep.Name, depSpecifier, IsOptionalPeer: false)); + } + } + + // Optional peer dependencies may appear as element[3] in bun.lock v1 array format. + if (element.GetArrayLength() > 3 && element[3].ValueKind == JsonValueKind.Object) + { + foreach (var dep in element[3].EnumerateObject()) + { + var depSpecifier = dep.Value.ValueKind == JsonValueKind.String ? dep.Value.GetString() : null; + dependencies.Add(new BunLockDependency(dep.Name, depSpecifier, IsOptionalPeer: true)); } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockScopeClassifier.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockScopeClassifier.cs new file mode 100644 index 000000000..8c7c1e661 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunLockScopeClassifier.cs @@ -0,0 +1,203 @@ +using System.Collections.Immutable; + +namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal; + +internal static class BunLockScopeClassifier +{ + public static BunLockData Classify(BunLockData lockData, IReadOnlyList declaredDependencies) + { + ArgumentNullException.ThrowIfNull(lockData); + ArgumentNullException.ThrowIfNull(declaredDependencies); + + if (lockData.AllEntries.IsEmpty || declaredDependencies.Count == 0) + { + return lockData; + } + + var entries = lockData.AllEntries.ToArray(); + + var entriesByName = entries + .GroupBy(static entry => entry.Name, StringComparer.Ordinal) + .ToDictionary(static group => group.Key, static group => group.ToImmutableArray(), StringComparer.Ordinal); + + var ambiguousNames = entriesByName + .Where(static pair => pair.Value.Length > 1) + .Select(static pair => pair.Key) + .ToHashSet(StringComparer.Ordinal); + + var hasAmbiguity = ambiguousNames.Count > 0; + + var uniqueByName = entriesByName + .Where(static pair => pair.Value.Length == 1) + .ToDictionary(static pair => pair.Key, static pair => pair.Value[0], StringComparer.Ordinal); + + var entryByKey = entries.ToDictionary(static entry => $"{entry.Name}@{entry.Version}", StringComparer.Ordinal); + + var prodRoots = ResolveDeclaredRoots(declaredDependencies, "prod", uniqueByName); + var devRoots = ResolveDeclaredRoots(declaredDependencies, "dev", uniqueByName); + var optionalRoots = ResolveDeclaredRoots(declaredDependencies, "optional", uniqueByName); + var peerRoots = ResolveDeclaredRoots(declaredDependencies, "peer", uniqueByName); + + var prodReachable = Traverse(prodRoots, uniqueByName, entryByKey, includeOptionalPeer: false); + var devReachable = Traverse(devRoots, uniqueByName, entryByKey, includeOptionalPeer: false); + var optionalReachable = Traverse(optionalRoots, uniqueByName, entryByKey, includeOptionalPeer: false); + var peerReachable = Traverse(peerRoots, uniqueByName, entryByKey, includeOptionalPeer: false); + + var baseReachable = new HashSet(StringComparer.Ordinal); + baseReachable.UnionWith(prodReachable); + baseReachable.UnionWith(devReachable); + baseReachable.UnionWith(optionalReachable); + baseReachable.UnionWith(peerReachable); + + // Optional peer edges (when present) promote targets into both optional and peer scopes. + var optionalPeerRoots = CollectOptionalPeerTargets(baseReachable, entryByKey, uniqueByName); + if (optionalPeerRoots.Count > 0) + { + var optionalPeerReachable = Traverse(optionalPeerRoots, uniqueByName, entryByKey, includeOptionalPeer: false); + optionalReachable.UnionWith(optionalPeerReachable); + peerReachable.UnionWith(optionalPeerReachable); + } + + var rewritten = ImmutableArray.CreateBuilder(entries.Length); + foreach (var entry in entries) + { + var key = $"{entry.Name}@{entry.Version}"; + + var reachedFromProd = prodReachable.Contains(key); + var reachedFromDev = devReachable.Contains(key); + var reachedFromOptional = optionalReachable.Contains(key); + var reachedFromPeer = peerReachable.Contains(key); + + var computedDev = !hasAmbiguity && reachedFromDev && !reachedFromProd; + var computedOptional = reachedFromOptional; + var computedPeer = reachedFromPeer; + + var computedScopeUnknown = ambiguousNames.Contains(entry.Name) + || (hasAmbiguity && (reachedFromDev || reachedFromOptional || reachedFromPeer) && !reachedFromProd); + + rewritten.Add(new BunLockEntry + { + Name = entry.Name, + Version = entry.Version, + Resolved = entry.Resolved, + Integrity = entry.Integrity, + IsDev = entry.IsDev || computedDev, + IsOptional = entry.IsOptional || computedOptional, + IsPeer = entry.IsPeer || computedPeer, + ScopeUnknown = entry.ScopeUnknown || computedScopeUnknown, + SourceType = entry.SourceType, + GitCommit = entry.GitCommit, + Specifier = entry.Specifier, + Dependencies = entry.Dependencies + }); + } + + return new BunLockData(rewritten.ToImmutable()); + } + + private static IReadOnlyList ResolveDeclaredRoots( + IReadOnlyList declared, + string scope, + IReadOnlyDictionary uniqueByName) + { + var roots = new List(); + foreach (var dep in declared) + { + if (!scope.Equals(dep.Scope, StringComparison.Ordinal)) + { + continue; + } + + if (uniqueByName.TryGetValue(dep.Name, out var entry)) + { + roots.Add(entry); + } + } + + return roots; + } + + private static HashSet Traverse( + IReadOnlyList roots, + IReadOnlyDictionary uniqueByName, + IReadOnlyDictionary entryByKey, + bool includeOptionalPeer) + { + var visited = new HashSet(StringComparer.Ordinal); + var queue = new Queue(); + + foreach (var root in roots) + { + var key = $"{root.Name}@{root.Version}"; + if (visited.Add(key)) + { + queue.Enqueue(root); + } + } + + while (queue.Count > 0) + { + var current = queue.Dequeue(); + foreach (var dep in current.Dependencies) + { + if (!includeOptionalPeer && dep.IsOptionalPeer) + { + continue; + } + + if (!uniqueByName.TryGetValue(dep.Name, out var target)) + { + continue; // Unknown or ambiguous dependency. + } + + var key = $"{target.Name}@{target.Version}"; + if (visited.Add(key) && entryByKey.TryGetValue(key, out var resolved)) + { + queue.Enqueue(resolved); + } + } + } + + return visited; + } + + private static IReadOnlyList CollectOptionalPeerTargets( + IEnumerable reachableKeys, + IReadOnlyDictionary entryByKey, + IReadOnlyDictionary uniqueByName) + { + var targets = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var key in reachableKeys) + { + if (!entryByKey.TryGetValue(key, out var entry)) + { + continue; + } + + foreach (var dep in entry.Dependencies.Where(static dep => dep.IsOptionalPeer)) + { + if (!uniqueByName.TryGetValue(dep.Name, out var target)) + { + continue; + } + + var targetKey = $"{target.Name}@{target.Version}"; + if (seen.Add(targetKey)) + { + targets.Add(target); + } + } + } + + targets.Sort(static (left, right) => + { + var nameCompare = string.CompareOrdinal(left.Name, right.Name); + return nameCompare != 0 ? nameCompare : string.CompareOrdinal(left.Version, right.Version); + }); + + return targets; + } +} + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunPackage.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunPackage.cs index 195cc6880..089b61ec8 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunPackage.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunPackage.cs @@ -14,7 +14,7 @@ internal sealed class BunPackage { Name = name; Version = version; - Purl = BuildPurl(name, version); + Purl = BuildNpmPurl(name, version); ComponentKey = $"purl::{Purl}"; } @@ -29,6 +29,12 @@ internal sealed class BunPackage public bool IsDev { get; private init; } public bool IsOptional { get; private init; } public bool IsPeer { get; private init; } + public bool ScopeUnknown { get; private init; } + public string? PackageJsonSha256 { get; private init; } + public string? PackageJsonHashSkipReason { get; private init; } + public string? LockfilePath { get; set; } + public string? LockfileSha256 { get; set; } + public string? LockfileHashSkipReason { get; set; } /// /// Source type: npm, git, tarball, file, link, workspace. @@ -99,7 +105,9 @@ internal sealed class BunPackage string logicalPath, string? realPath, bool isPrivate, - BunLockEntry? lockEntry) + BunLockEntry? lockEntry, + string? packageJsonSha256 = null, + string? packageJsonHashSkipReason = null) { return new BunPackage(name, version) { @@ -112,10 +120,15 @@ internal sealed class BunPackage IsDev = lockEntry?.IsDev ?? false, IsOptional = lockEntry?.IsOptional ?? false, IsPeer = lockEntry?.IsPeer ?? false, + ScopeUnknown = lockEntry?.ScopeUnknown ?? false, SourceType = lockEntry?.SourceType ?? "npm", GitCommit = lockEntry?.GitCommit, Specifier = lockEntry?.Specifier, - Dependencies = lockEntry?.Dependencies ?? Array.Empty() + PackageJsonSha256 = packageJsonSha256, + PackageJsonHashSkipReason = packageJsonHashSkipReason, + Dependencies = lockEntry is null + ? Array.Empty() + : lockEntry.Dependencies.Select(static dep => dep.Name).ToArray() }; } @@ -131,10 +144,11 @@ internal sealed class BunPackage IsDev = entry.IsDev, IsOptional = entry.IsOptional, IsPeer = entry.IsPeer, + ScopeUnknown = entry.ScopeUnknown, SourceType = entry.SourceType, GitCommit = entry.GitCommit, Specifier = entry.Specifier, - Dependencies = entry.Dependencies + Dependencies = entry.Dependencies.Select(static dep => dep.Name).ToArray() }; } @@ -172,6 +186,18 @@ internal sealed class BunPackage metadata["private"] = "true"; } + if (!string.IsNullOrEmpty(PackageJsonHashSkipReason)) + { + metadata["packageJson.hashSkipped"] = "true"; + metadata["packageJson.hashSkipReason"] = PackageJsonHashSkipReason; + } + + if (!string.IsNullOrEmpty(LockfileHashSkipReason)) + { + metadata["bunLock.hashSkipped"] = "true"; + metadata["bunLock.hashSkipReason"] = LockfileHashSkipReason; + } + if (!string.IsNullOrEmpty(CustomRegistry)) { metadata["customRegistry"] = CustomRegistry; @@ -182,6 +208,11 @@ internal sealed class BunPackage metadata["dev"] = "true"; } + if (ScopeUnknown) + { + metadata["scopeUnknown"] = "true"; + } + if (IsDirect) { metadata["direct"] = "true"; @@ -243,36 +274,41 @@ internal sealed class BunPackage Source ?? "node_modules", NormalizePath(Path.Combine(LogicalPath, "package.json")), null, - null)); + PackageJsonSha256)); } if (!string.IsNullOrEmpty(Resolved)) { + var locator = BuildLockLocator(); evidence.Add(new LanguageComponentEvidence( LanguageEvidenceKind.Metadata, "resolved", - "bun.lock", + locator, Resolved, - null)); + LockfileSha256)); } if (!string.IsNullOrEmpty(Integrity)) { + var locator = BuildLockLocator(); evidence.Add(new LanguageComponentEvidence( LanguageEvidenceKind.Metadata, "integrity", - "bun.lock", + locator, Integrity, - null)); + LockfileSha256)); } return evidence; } - private static string BuildPurl(string name, string version) + private string BuildLockLocator() + => $"{NormalizePath(string.IsNullOrWhiteSpace(LockfilePath) ? "bun.lock" : LockfilePath!)}:packages[{Name}@{Version}]"; + + internal static string BuildNpmPurl(string name, string version) { // pkg:npm/@ - // Scoped packages: @scope/name → %40scope/name + // Scoped packages: @scope/name -> %40scope/name var encodedName = name.StartsWith('@') ? $"%40{HttpUtility.UrlEncode(name[1..]).Replace("%2f", "/", StringComparison.OrdinalIgnoreCase)}" : HttpUtility.UrlEncode(name); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunProjectDiscoverer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunProjectDiscoverer.cs index eb03254bc..71e4aa119 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunProjectDiscoverer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunProjectDiscoverer.cs @@ -26,12 +26,26 @@ internal static class BunProjectDiscoverer { ArgumentNullException.ThrowIfNull(context); - var roots = new List(); - DiscoverRecursive(context.RootPath, 0, roots, cancellationToken); + var roots = new List(capacity: 8); + var unique = new HashSet(StringComparer.Ordinal); + + foreach (var discoveryRoot in EnumerateDiscoveryRoots(context.RootPath, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + DiscoverRecursive(discoveryRoot, 0, roots, unique, cancellationToken); + + if (roots.Count >= MaxRoots) + { + break; + } + } + + roots.Sort(StringComparer.Ordinal); return roots.ToImmutableArray(); } - private static void DiscoverRecursive(string directory, int depth, List roots, CancellationToken cancellationToken) + private static void DiscoverRecursive(string directory, int depth, List roots, HashSet unique, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -48,7 +62,11 @@ internal static class BunProjectDiscoverer // Check if this directory is a Bun project root if (IsBunProjectRoot(directory)) { - roots.Add(directory); + if (unique.Add(directory)) + { + roots.Add(directory); + } + // Don't recurse into node_modules or .bun return; } @@ -56,7 +74,7 @@ internal static class BunProjectDiscoverer // Recurse into subdirectories try { - foreach (var subdir in Directory.EnumerateDirectories(directory)) + foreach (var subdir in EnumerateDirectoriesSorted(directory)) { cancellationToken.ThrowIfCancellationRequested(); @@ -68,7 +86,7 @@ internal static class BunProjectDiscoverer continue; } - DiscoverRecursive(subdir, depth + 1, roots, cancellationToken); + DiscoverRecursive(subdir, depth + 1, roots, unique, cancellationToken); if (roots.Count >= MaxRoots) { @@ -117,7 +135,108 @@ internal static class BunProjectDiscoverer private static bool ShouldSkipDirectory(string dirName) { - return dirName is "node_modules" or ".git" or ".svn" or ".hg" or "bin" or "obj" or ".bun" - || dirName.StartsWith('.'); // Skip hidden directories + if (dirName is "node_modules" or ".git" or ".svn" or ".hg" or "bin" or "obj" or ".bun") + { + return true; + } + + // Do not skip container layer roots like ".layers". + if (dirName.Equals(".layers", StringComparison.Ordinal)) + { + return false; + } + + // Skip other hidden directories by default. + return dirName.Length > 0 && dirName[0] == '.'; + } + + private static IEnumerable EnumerateDiscoveryRoots(string rootPath, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(rootPath)) + { + yield break; + } + + var fullRoot = Path.GetFullPath(rootPath); + if (!Directory.Exists(fullRoot)) + { + yield break; + } + + yield return fullRoot; + + foreach (var containerRoot in EnumerateContainerLayerRoots(fullRoot, cancellationToken)) + { + yield return containerRoot; + } + } + + private static IEnumerable EnumerateContainerLayerRoots(string rootPath, CancellationToken cancellationToken) + { + var candidates = new List(); + + // Common unpack layouts: + // - layers//... + // - .layers//... + // - layer0/... (direct children) + var layersRoot = Path.Combine(rootPath, "layers"); + if (Directory.Exists(layersRoot)) + { + candidates.AddRange(EnumerateDirectoriesSorted(layersRoot)); + } + + var dotLayersRoot = Path.Combine(rootPath, ".layers"); + if (Directory.Exists(dotLayersRoot)) + { + candidates.AddRange(EnumerateDirectoriesSorted(dotLayersRoot)); + } + + foreach (var directChild in EnumerateDirectoriesSorted(rootPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var name = Path.GetFileName(directChild); + if (name is null) + { + continue; + } + + if (name.StartsWith("layer", StringComparison.OrdinalIgnoreCase) && + !name.Equals("layers", StringComparison.OrdinalIgnoreCase) && + !name.Equals(".layers", StringComparison.OrdinalIgnoreCase)) + { + candidates.Add(directChild); + } + } + + candidates.Sort(StringComparer.Ordinal); + + foreach (var candidate in candidates) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (Directory.Exists(candidate)) + { + yield return candidate; + } + } + } + + private static IReadOnlyList EnumerateDirectoriesSorted(string directory) + { + try + { + var entries = Directory.EnumerateDirectories(directory).ToList(); + entries.Sort(StringComparer.Ordinal); + return entries; + } + catch (UnauthorizedAccessException) + { + return Array.Empty(); + } + catch (DirectoryNotFoundException) + { + return Array.Empty(); + } } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunVersionSpec.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunVersionSpec.cs new file mode 100644 index 000000000..e8d196219 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunVersionSpec.cs @@ -0,0 +1,144 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Bun.Internal; + +internal static class BunVersionSpec +{ + public static bool IsConcreteNpmVersion(string? version) + { + if (string.IsNullOrWhiteSpace(version)) + { + return false; + } + + version = version.Trim(); + if (version.StartsWith('^') || version.StartsWith('~') || version.StartsWith('>') || version.StartsWith('<')) + { + return false; + } + + if (version.Contains(' ') || version.Contains('*') || version.Contains('|') || version.Contains(':') || version.Contains('/')) + { + return false; + } + + var hasDigit = false; + foreach (var ch in version) + { + if (char.IsAsciiDigit(ch)) + { + hasDigit = true; + continue; + } + + if (ch is '.' or '-' or '+' or '_' || char.IsAsciiLetter(ch)) + { + continue; + } + + return false; + } + + return hasDigit; + } + + public static bool IsConcreteSemver(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var span = value.AsSpan().Trim(); + if (span.Length == 0 || !char.IsAsciiDigit(span[0])) + { + return false; + } + + var index = 0; + if (!ConsumeDigits(span, ref index)) + { + return false; + } + + if (!ConsumeChar(span, ref index, '.')) + { + return false; + } + + if (!ConsumeDigits(span, ref index)) + { + return false; + } + + if (!ConsumeChar(span, ref index, '.')) + { + return false; + } + + if (!ConsumeDigits(span, ref index)) + { + return false; + } + + // Optional prerelease: -[0-9A-Za-z.-]+ + if (index < span.Length && span[index] == '-') + { + index++; + if (!ConsumeSemverIdentifiers(span, ref index)) + { + return false; + } + } + + // Optional build metadata: +[0-9A-Za-z.-]+ + if (index < span.Length && span[index] == '+') + { + index++; + if (!ConsumeSemverIdentifiers(span, ref index)) + { + return false; + } + } + + return index == span.Length; + } + + private static bool ConsumeDigits(ReadOnlySpan span, ref int index) + { + var start = index; + while (index < span.Length && char.IsAsciiDigit(span[index])) + { + index++; + } + + return index > start; + } + + private static bool ConsumeChar(ReadOnlySpan span, ref int index, char expected) + { + if (index >= span.Length || span[index] != expected) + { + return false; + } + + index++; + return true; + } + + private static bool ConsumeSemverIdentifiers(ReadOnlySpan span, ref int index) + { + var start = index; + while (index < span.Length) + { + var ch = span[index]; + if (char.IsAsciiLetterOrDigit(ch) || ch is '.' or '-') + { + index++; + continue; + } + + break; + } + + return index > start; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunWorkspaceHelper.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunWorkspaceHelper.cs index 4e61fbddf..8ce32d1f0 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunWorkspaceHelper.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/Internal/BunWorkspaceHelper.cs @@ -317,9 +317,9 @@ internal static class BunWorkspaceHelper var patchFile = entry.Value.GetString(); if (!string.IsNullOrEmpty(patchFile)) { - // Parse package name from key (could be "pkg@version" or just "pkg") - var packageName = ExtractPackageName(entry.Name); - result[packageName] = patchFile; + // Preserve version specificity (name@version) when present. + var patchKey = NormalizePatchKey(entry.Name); + result[patchKey] = NormalizePatchPath(projectRoot, patchFile); } } } @@ -328,31 +328,32 @@ internal static class BunWorkspaceHelper var patchesDir = Path.Combine(projectRoot, "patches"); if (Directory.Exists(patchesDir)) { - ScanPatchesDirectory(patchesDir, result); + ScanPatchesDirectory(patchesDir, projectRoot, result); } // Bun uses .patches directory var bunPatchesDir = Path.Combine(projectRoot, ".patches"); if (Directory.Exists(bunPatchesDir)) { - ScanPatchesDirectory(bunPatchesDir, result); + ScanPatchesDirectory(bunPatchesDir, projectRoot, result); } return result; } - private static void ScanPatchesDirectory(string patchesDir, Dictionary result) + private static void ScanPatchesDirectory(string patchesDir, string projectRoot, Dictionary result) { try { - foreach (var patchFile in Directory.EnumerateFiles(patchesDir, "*.patch")) + foreach (var patchFile in Directory.EnumerateFiles(patchesDir, "*.patch").OrderBy(static path => Path.GetFileName(path), StringComparer.Ordinal)) { - // Patch file name format: package-name@version.patch + // Patch file name format (pnpm/bun): package-name@version.patch or @scope+name@version.patch var fileName = Path.GetFileNameWithoutExtension(patchFile); - var packageName = ExtractPackageName(fileName); - if (!string.IsNullOrEmpty(packageName) && !result.ContainsKey(packageName)) + var patchKey = NormalizePatchKey(fileName); + if (!string.IsNullOrEmpty(patchKey) && !result.ContainsKey(patchKey)) { - result[packageName] = patchFile; + var relative = Path.GetRelativePath(projectRoot, patchFile); + result[patchKey] = NormalizePath(relative); } } } @@ -362,30 +363,63 @@ internal static class BunWorkspaceHelper } } - private static string ExtractPackageName(string nameWithVersion) + private static string NormalizePatchKey(string nameWithVersion) { - // Format: package-name@version or @scope/package-name@version - if (string.IsNullOrEmpty(nameWithVersion)) + if (string.IsNullOrWhiteSpace(nameWithVersion)) { return string.Empty; } - // For scoped packages, find @ after the scope - if (nameWithVersion.StartsWith('@')) + var trimmed = nameWithVersion.Trim(); + + // pnpm patch naming encodes @scope/name as @scope+name + if (trimmed.StartsWith('@') && !trimmed.Contains('/', StringComparison.Ordinal)) { - var slashIndex = nameWithVersion.IndexOf('/'); - if (slashIndex > 0) + // Replace the first '+' in the name portion (before the version delimiter) with '/' + var versionAt = trimmed.IndexOf('@', 1); + var namePart = versionAt > 0 ? trimmed[..versionAt] : trimmed; + var versionPart = versionAt > 0 ? trimmed[versionAt..] : string.Empty; + + var plusIndex = namePart.IndexOf('+', StringComparison.Ordinal); + if (plusIndex > 0) { - var atIndex = nameWithVersion.IndexOf('@', slashIndex); - return atIndex > slashIndex ? nameWithVersion[..atIndex] : nameWithVersion; + namePart = $"{namePart[..plusIndex]}/{namePart[(plusIndex + 1)..]}"; + } + + return namePart + versionPart; + } + + return trimmed; + } + + private static string NormalizePatchPath(string projectRoot, string patchFile) + { + if (string.IsNullOrWhiteSpace(patchFile)) + { + return string.Empty; + } + + var trimmed = patchFile.Trim(); + + // Avoid absolute path leakage: convert to project-relative when possible. + if (Path.IsPathRooted(trimmed)) + { + try + { + trimmed = Path.GetRelativePath(projectRoot, trimmed); + } + catch + { + // Keep the original string if we cannot relativize (still scrubbed later by metadata rules). } } - // For regular packages - var lastAtIndex = nameWithVersion.LastIndexOf('@'); - return lastAtIndex > 0 ? nameWithVersion[..lastAtIndex] : nameWithVersion; + return NormalizePath(trimmed); } + private static string NormalizePath(string path) + => path.Replace('\\', '/'); + private static void AddDependencies( JsonElement root, string propertyName, diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/TASKS.md b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/TASKS.md new file mode 100644 index 000000000..e3dffbe44 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Bun/TASKS.md @@ -0,0 +1,13 @@ +# Bun Analyzer Tasks (Sprint 0407) + +| Task ID | Status | Notes | Updated (UTC) | +| --- | --- | --- | --- | +| SCAN-BUN-407-001 | DONE | Container-layer aware project discovery (`layers/`, `.layers/`, `layer*`), bounded + deterministic. | 2025-12-13 | +| SCAN-BUN-407-002 | DONE | Declared-only fallback from `package.json` with safe identities (no range-as-version PURLs). | 2025-12-13 | +| SCAN-BUN-407-003 | DONE | bun.lock v1 graph enrichment (dependency specifiers + deterministic dev/optional/peer classification). | 2025-12-13 | +| SCAN-BUN-407-004 | DONE | Make `includeDev` meaningful for lockfile-only and installed scans; use `scopeUnknown` when unsure. | 2025-12-13 | +| SCAN-BUN-407-005 | DONE | Version-specific patch mapping + relative patch paths (no absolute path leakage). | 2025-12-13 | +| SCAN-BUN-407-006 | DONE | Evidence strengthening + locator precision (bun.lock locators, bounded sha256). | 2025-12-13 | +| SCAN-BUN-407-007 | DONE | Identity safety for non-npm sources (git/file/link/workspace/tarball/custom registry). | 2025-12-13 | +| SCAN-BUN-407-008 | DONE | Document analyzer contract under `docs/modules/scanner/` and link sprint. | 2025-12-13 | +| SCAN-BUN-407-009 | DONE | Optional: deterministic benchmark if perf risk materializes. | 2025-12-13 | diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoBundleInspector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoBundleInspector.cs index 5b50d4c3c..bf5b204b8 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoBundleInspector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoBundleInspector.cs @@ -21,6 +21,7 @@ internal static class DenoBundleInspector } sourcePath ??= "(stream)"; + sourcePath = sourcePath.Replace('\\', '/'); try { diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoContainerAdapter.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoContainerAdapter.cs index 8601c9ce8..b46b01642 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoContainerAdapter.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoContainerAdapter.cs @@ -15,11 +15,14 @@ internal static class DenoContainerAdapter private static void AddCaches(DenoWorkspace workspace, ImmutableArray.Builder builder) { - foreach (var cache in workspace.CacheLocations) + foreach (var cache in workspace.CacheLocations + .OrderByDescending(static cache => !string.IsNullOrWhiteSpace(cache.LayerDigest)) + .ThenBy(static cache => cache.Kind) + .ThenBy(static cache => cache.AbsolutePath, StringComparer.OrdinalIgnoreCase)) { var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) { - ["path"] = cache.AbsolutePath, + ["path"] = NormalizePath(cache.AbsolutePath), ["alias"] = cache.Alias, ["kind"] = cache.Kind.ToString() }; @@ -35,11 +38,13 @@ internal static class DenoContainerAdapter private static void AddVendors(DenoWorkspace workspace, ImmutableArray.Builder builder) { - foreach (var vendor in workspace.Vendors) + foreach (var vendor in workspace.Vendors + .OrderBy(static vendor => !string.IsNullOrWhiteSpace(vendor.LayerDigest)) + .ThenBy(static vendor => vendor.RelativePath, StringComparer.Ordinal)) { var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) { - ["path"] = vendor.AbsolutePath, + ["path"] = NormalizePath(vendor.AbsolutePath), ["alias"] = vendor.Alias }; @@ -73,4 +78,7 @@ internal static class DenoContainerAdapter bundle)); } } + + private static string NormalizePath(string value) + => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Replace('\\', '/'); } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoWorkspaceNormalizer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoWorkspaceNormalizer.cs index 5d8892059..0e4e5af63 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoWorkspaceNormalizer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/DenoWorkspaceNormalizer.cs @@ -195,7 +195,7 @@ internal static class DenoWorkspaceNormalizer cancellationToken.ThrowIfCancellationRequested(); var relative = context.GetRelativePath(absolute); - var alias = DenoPathUtilities.CreateAlias(absolute, "vendor"); + var alias = DenoPathUtilities.CreateAlias(relative, "vendor"); var layerDigest = DenoLayerMetadata.TryExtractDigest(absolute); DenoImportMapDocument? importMap = null; @@ -272,7 +272,7 @@ internal static class DenoWorkspaceNormalizer cancellationToken.ThrowIfCancellationRequested(); - var alias = DenoPathUtilities.CreateAlias(absolute, "deno"); + var alias = DenoPathUtilities.CreateAlias(context.GetRelativePath(absolute), "deno"); builder.Add(new DenoCacheLocation( absolute, alias, diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/Runtime/DenoRuntimeTraceSerializer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/Runtime/DenoRuntimeTraceSerializer.cs index 24baa6c1c..eb1cd8f64 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/Runtime/DenoRuntimeTraceSerializer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Deno/Internal/Runtime/DenoRuntimeTraceSerializer.cs @@ -23,14 +23,12 @@ internal static class DenoRuntimeTraceSerializer .ToArray(); using var stream = new MemoryStream(); - using (var writer = new Utf8JsonWriter(stream, WriterOptions)) + foreach (var evt in ordered) { - foreach (var evt in ordered) - { - WriteEvent(writer, evt); - writer.Flush(); - stream.WriteByte((byte)'\n'); - } + using var writer = new Utf8JsonWriter(stream, WriterOptions); + WriteEvent(writer, evt); + writer.Flush(); + stream.WriteByte((byte)'\n'); } var bytes = stream.ToArray(); @@ -136,12 +134,24 @@ internal static class DenoRuntimeTraceSerializer { if (!string.IsNullOrWhiteSpace(p)) { - permissions.Add(p.Trim().ToLowerInvariant()); + var normalized = p.Trim().ToLowerInvariant(); + if (!string.Equals(normalized, "unknown", StringComparison.Ordinal)) + { + permissions.Add(normalized); + } } } break; - case DenoPermissionUseEvent: + case DenoPermissionUseEvent e: permissionUses++; + if (!string.IsNullOrWhiteSpace(e.Permission)) + { + var normalized = e.Permission.Trim().ToLowerInvariant(); + if (!string.Equals(normalized, "unknown", StringComparison.Ordinal)) + { + permissions.Add(normalized); + } + } break; case DenoNpmResolutionEvent: npmResolutions++; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.DotNet/Internal/DotNetEntrypointResolver.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.DotNet/Internal/DotNetEntrypointResolver.cs index a6cd6fbb9..b50684cfe 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.DotNet/Internal/DotNetEntrypointResolver.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.DotNet/Internal/DotNetEntrypointResolver.cs @@ -45,8 +45,10 @@ public static class DotNetEntrypointResolver continue; } + var name = GetEntrypointName(depsPath); + DotNetRuntimeConfig? runtimeConfig = null; - var runtimeConfigPath = Path.ChangeExtension(depsPath, ".runtimeconfig.json"); + var runtimeConfigPath = GetRuntimeConfigPath(depsPath, name); string? relativeRuntimeConfig = null; if (!string.IsNullOrEmpty(runtimeConfigPath) && File.Exists(runtimeConfigPath)) @@ -59,7 +61,6 @@ public static class DotNetEntrypointResolver var rids = CollectRuntimeIdentifiers(depsFile, runtimeConfig); var publishKind = DeterminePublishKind(depsFile); - var name = GetEntrypointName(depsPath); var id = BuildDeterministicId(name, tfms, rids, publishKind); results.Add(new DotNetEntrypoint( @@ -101,6 +102,19 @@ public static class DotNetEntrypointResolver return stem; } + private static string GetRuntimeConfigPath(string depsPath, string entrypointName) + { + var directory = Path.GetDirectoryName(depsPath); + var fileName = $"{entrypointName}.runtimeconfig.json"; + + if (string.IsNullOrWhiteSpace(directory)) + { + return fileName; + } + + return Path.Combine(directory, fileName); + } + private static IReadOnlyCollection CollectTargetFrameworks(DotNetDepsFile depsFile, DotNetRuntimeConfig? runtimeConfig) { var tfms = new SortedSet(StringComparer.OrdinalIgnoreCase); @@ -109,7 +123,11 @@ public static class DotNetEntrypointResolver { foreach (var tfm in library.TargetFrameworks) { - tfms.Add(tfm); + var normalized = NormalizeTargetFrameworkMoniker(tfm); + if (!string.IsNullOrWhiteSpace(normalized)) + { + tfms.Add(normalized); + } } } @@ -129,6 +147,83 @@ public static class DotNetEntrypointResolver return tfms; } + private static string? NormalizeTargetFrameworkMoniker(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + + if (TryNormalizeFrameworkMoniker(trimmed, ".NETCoreApp,Version=v", "net", out var normalized)) + { + return normalized; + } + + if (TryNormalizeFrameworkMoniker(trimmed, ".NETStandard,Version=v", "netstandard", out normalized)) + { + return normalized; + } + + if (TryNormalizeFrameworkMoniker(trimmed, ".NETFramework,Version=v", "net", out normalized)) + { + return NormalizeNetFrameworkTfm(normalized!); + } + + return trimmed; + } + + private static bool TryNormalizeFrameworkMoniker(string value, string prefix, string replacement, out string? normalized) + { + normalized = null; + + if (!value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var versionPart = value[prefix.Length..]; + if (string.IsNullOrWhiteSpace(versionPart)) + { + return false; + } + + versionPart = versionPart.Trim(); + if (!Version.TryParse(versionPart, out var version)) + { + return false; + } + + normalized = $"{replacement}{version.Major}.{version.Minor}"; + return true; + } + + private static string NormalizeNetFrameworkTfm(string value) + { + if (!value.StartsWith("net", StringComparison.OrdinalIgnoreCase)) + { + return value; + } + + var versionPart = value[3..]; + if (!Version.TryParse(versionPart, out var version)) + { + return value.Replace(".", string.Empty, StringComparison.Ordinal); + } + + var major = Math.Max(version.Major, 0); + var minor = Math.Max(version.Minor, 0); + var build = version.Build; + + if (build > 0) + { + return $"net{major}{minor}{build}"; + } + + return $"net{major}{minor}"; + } + private static IReadOnlyCollection CollectRuntimeIdentifiers(DotNetDepsFile depsFile, DotNetRuntimeConfig? runtimeConfig) { var rids = new SortedSet(StringComparer.OrdinalIgnoreCase); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/GoLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/GoLanguageAnalyzer.cs index f1059c9d1..dd0821545 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/GoLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/GoLanguageAnalyzer.cs @@ -100,7 +100,7 @@ public sealed class GoLanguageAnalyzer : ILanguageAnalyzer cancellationToken.ThrowIfCancellationRequested(); // Quick check for known binary formats - if (GoBinaryFormatDetector.IsPotentialBinary(path)) + if (GoBinaryFormatDetector.IsPotentialBinary(path) || GoBinaryScanner.HasBuildInfoMagicPrefix(path)) { candidatePaths.Add(path); } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoBinaryScanner.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoBinaryScanner.cs index cc1f5774f..6b18a201f 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoBinaryScanner.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoBinaryScanner.cs @@ -33,6 +33,45 @@ internal static class GoBinaryScanner } } + public static bool HasBuildInfoMagicPrefix(string filePath) + { + if (string.IsNullOrWhiteSpace(filePath)) + { + return false; + } + + try + { + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); + if (stream.Length < BuildInfoMagic.Length) + { + return false; + } + + var size = BuildInfoMagic.Length; + Span header = stackalloc byte[size]; + var read = stream.Read(header); + if (read != header.Length) + { + return false; + } + + return header.SequenceEqual(BuildInfoMagic.Span); + } + catch (IOException) + { + return false; + } + catch (UnauthorizedAccessException) + { + return false; + } + catch (System.Security.SecurityException) + { + return false; + } + } + public static bool TryReadBuildInfo(string filePath, out string? goVersion, out string? moduleData) { goVersion = null; diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCapabilityScanner.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCapabilityScanner.cs index 505b7d335..f69fc3bf2 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCapabilityScanner.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCapabilityScanner.cs @@ -36,8 +36,8 @@ internal static partial class GoCapabilityScanner var originalLine = lineIndex < lines.Length ? lines[lineIndex] : strippedLine; var lineNumber = lineIndex + 1; - // Skip empty lines - if (string.IsNullOrWhiteSpace(strippedLine)) + // Skip whitespace-only lines while still scanning comment-only lines for directives. + if (string.IsNullOrWhiteSpace(strippedLine) && string.IsNullOrWhiteSpace(originalLine)) { continue; } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCgoDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCgoDetector.cs index 066d7e05a..a05c95dd3 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCgoDetector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoCgoDetector.cs @@ -387,7 +387,7 @@ internal static partial class GoCgoDetector /// Matches #cgo directives with optional build constraints. /// Format: #cgo [GOOS GOARCH] DIRECTIVE: value /// - [GeneratedRegex(@"#cgo\s+(?:([a-z0-9_,!\s]+)\s+)?(\w+):\s*(.+?)(?=\n|$)", RegexOptions.Multiline | RegexOptions.IgnoreCase)] + [GeneratedRegex(@"#cgo\s+(?:([a-z0-9_,!\s]+)\s+)?([\w-]+):\s*(.+?)(?=\n|$)", RegexOptions.Multiline | RegexOptions.IgnoreCase)] private static partial Regex CgoDirectivePattern(); /// diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoLicenseDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoLicenseDetector.cs index 3b704caf0..9e5581f80 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoLicenseDetector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoLicenseDetector.cs @@ -44,6 +44,9 @@ internal static partial class GoLicenseDetector new("Apache-1.1", @"Apache License.*?(?:Version 1\.1|v1\.1)", "Apache License, Version 1.1"), new("Apache-1.0", @"Apache License.*?(?:Version 1\.0|v1\.0)", "Apache License, Version 1.0"), + // Boost (avoid mis-classifying as MIT) + new("BSL-1.0", @"Boost Software License", "Boost Software License 1.0"), + // MIT variants new("MIT", @"(?:MIT License|Permission is hereby granted, free of charge)", "MIT License"), new("MIT-0", @"MIT No Attribution", "MIT No Attribution"), @@ -82,7 +85,6 @@ internal static partial class GoLicenseDetector new("Unlicense", @"This is free and unencumbered software released into the public domain", "The Unlicense"), new("WTFPL", @"DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE", "Do What The F*ck You Want To Public License"), new("Zlib", @"zlib License|This software is provided 'as-is'", "zlib License"), - new("BSL-1.0", @"Boost Software License", "Boost Software License 1.0"), new("PostgreSQL", @"PostgreSQL License", "PostgreSQL License"), new("BlueOak-1.0.0", @"Blue Oak Model License", "Blue Oak Model License 1.0.0"), diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoVersionConflictDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoVersionConflictDetector.cs index d9d9b8f76..78a4bcffe 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoVersionConflictDetector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Go/Internal/GoVersionConflictDetector.cs @@ -361,7 +361,7 @@ internal static partial class GoVersionConflictDetector /// /// Matches pseudo-versions: v0.0.0-timestamp-hash or vX.Y.Z-pre.0.timestamp-hash /// - [GeneratedRegex(@"^v\d+\.\d+\.\d+(-[a-z0-9]+)?\.?\d*\.?\d{14}-[a-f0-9]{12}$", RegexOptions.IgnoreCase)] + [GeneratedRegex(@"^v\d+\.\d+\.\d+-(?:\d{14}|(?:[0-9a-z-]+\.)*0\.\d{14})-[a-f0-9]{12}$", RegexOptions.IgnoreCase)] private static partial Regex PseudoVersionPattern(); /// diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/Internal/Jni/JavaJniAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/Internal/Jni/JavaJniAnalyzer.cs index 49c10e532..6427a4f26 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/Internal/Jni/JavaJniAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/Internal/Jni/JavaJniAnalyzer.cs @@ -13,6 +13,10 @@ internal static class JavaJniAnalyzer { private const ushort AccNative = 0x0100; + private const int MaxEdges = 2000; + private const int MaxWarnings = 200; + private const int MaxClassesPerSegment = 5000; + // Method references for System.load/loadLibrary and Runtime.load/loadLibrary private static readonly (string ClassName, string MethodName, string Descriptor, JavaJniReason Reason)[] JniLoadMethods = [ @@ -38,11 +42,44 @@ internal static class JavaJniAnalyzer { cancellationToken.ThrowIfCancellationRequested(); - foreach (var kvp in segment.ClassLocations) + var classesScanned = 0; + foreach (var kvp in segment.ClassLocations.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) { var className = kvp.Key; var location = kvp.Value; + if (edges.Count >= MaxEdges) + { + if (warnings.Count < MaxWarnings) + { + warnings.Add(new JavaJniWarning( + SourceClass: "*", + SegmentIdentifier: segment.Identifier, + WarningCode: "JNI_EDGE_LIMIT_REACHED", + Message: $"JNI edge limit ({MaxEdges}) reached; output truncated.", + MethodName: string.Empty, + MethodDescriptor: string.Empty)); + } + + break; + } + + if (classesScanned++ >= MaxClassesPerSegment) + { + if (warnings.Count < MaxWarnings) + { + warnings.Add(new JavaJniWarning( + SourceClass: "*", + SegmentIdentifier: segment.Identifier, + WarningCode: "JNI_CLASS_LIMIT_REACHED", + Message: $"JNI class scan limit ({MaxClassesPerSegment}) reached for segment; output truncated.", + MethodName: string.Empty, + MethodDescriptor: string.Empty)); + } + + break; + } + try { using var stream = location.OpenClassStream(cancellationToken); @@ -55,6 +92,11 @@ internal static class JavaJniAnalyzer if (method.IsNative) { + if (edges.Count >= MaxEdges) + { + break; + } + edges.Add(new JavaJniEdge( SourceClass: className, SegmentIdentifier: segment.Identifier, @@ -65,26 +107,44 @@ internal static class JavaJniAnalyzer MethodDescriptor: method.Descriptor, InstructionOffset: -1, Details: "native method declaration")); + + if (edges.Count >= MaxEdges) + { + break; + } } // Analyze bytecode for System.load/loadLibrary calls if (method.Code is not null) { AnalyzeMethodCode(classFile, method, segment.Identifier, className, edges, warnings); + + if (edges.Count >= MaxEdges) + { + break; + } } } } catch (Exception ex) when (ex is not OperationCanceledException) { - warnings.Add(new JavaJniWarning( - SourceClass: className, - SegmentIdentifier: segment.Identifier, - WarningCode: "JNI_PARSE_ERROR", - Message: $"Failed to parse class file: {ex.Message}", - MethodName: string.Empty, - MethodDescriptor: string.Empty)); + if (warnings.Count < MaxWarnings) + { + warnings.Add(new JavaJniWarning( + SourceClass: className, + SegmentIdentifier: segment.Identifier, + WarningCode: "JNI_PARSE_ERROR", + Message: $"Failed to parse class file: {ex.Message}", + MethodName: string.Empty, + MethodDescriptor: string.Empty)); + } } } + + if (edges.Count >= MaxEdges) + { + break; + } } if (edges.Count == 0 && warnings.Count == 0) @@ -234,6 +294,11 @@ internal static class JavaJniAnalyzer string className, List edges) { + if (edges.Count >= MaxEdges) + { + return; + } + var methodRef = classFile.ConstantPool.ResolveMethodRef(methodRefIndex); if (methodRef is null) { diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs index cb9e0b112..8474d5b42 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs @@ -1,11 +1,16 @@ using System.Collections.Generic; using System.IO; using System.IO.Compression; +using System.Globalization; using System.Linq; using System.Text; +using System.Xml; +using System.Xml.Linq; using StellaOps.Scanner.Analyzers.Lang.Java.Internal; using StellaOps.Scanner.Analyzers.Lang.Java.Internal.BuildMetadata; +using StellaOps.Scanner.Analyzers.Lang.Java.Internal.ClassPath; using StellaOps.Scanner.Analyzers.Lang.Java.Internal.Conflicts; +using StellaOps.Scanner.Analyzers.Lang.Java.Internal.Jni; using StellaOps.Scanner.Analyzers.Lang.Java.Internal.Osgi; using StellaOps.Scanner.Analyzers.Lang.Java.Internal.Shading; @@ -17,6 +22,14 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer public string DisplayName => "Java/Maven Analyzer"; + private sealed record JniBytecodeSummary( + int EdgeCount, + int WarningCount, + int NativeMethodCount, + int LoadCallCount, + IReadOnlyList Reasons, + IReadOnlyList TargetLibraries); + public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(context); @@ -24,6 +37,7 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer var workspace = JavaWorkspaceNormalizer.Normalize(context, cancellationToken); var lockData = await JavaLockFileCollector.LoadAsync(context, cancellationToken).ConfigureAwait(false); + var jniBytecodeSummaries = BuildJniBytecodeSummaries(workspace, cancellationToken); var matchedLocks = new HashSet(StringComparer.OrdinalIgnoreCase); var hasLockEntries = lockData.HasEntries; @@ -33,7 +47,9 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer try { - await ProcessArchiveAsync(archive, context, writer, lockData, matchedLocks, hasLockEntries, cancellationToken).ConfigureAwait(false); + var archiveKey = NormalizeArchivePath(archive.RelativePath); + jniBytecodeSummaries.TryGetValue(archiveKey, out var jniSummary); + await ProcessArchiveAsync(archive, context, writer, lockData, matchedLocks, hasLockEntries, jniSummary, cancellationToken).ConfigureAwait(false); } catch (IOException) { @@ -75,6 +91,117 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer } } + private static IReadOnlyDictionary BuildJniBytecodeSummaries(JavaWorkspace workspace, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(workspace); + + var classPath = JavaClassPathBuilder.Build(workspace, cancellationToken); + if (classPath.Segments.IsDefaultOrEmpty) + { + return new Dictionary(StringComparer.Ordinal); + } + + var analysis = JavaJniAnalyzer.Analyze(classPath, cancellationToken); + if (analysis.Edges.IsDefaultOrEmpty && analysis.Warnings.IsDefaultOrEmpty) + { + return new Dictionary(StringComparer.Ordinal); + } + + const int MaxTargetLibraries = 12; + + var builders = new Dictionary(StringComparer.Ordinal); + + foreach (var edge in analysis.Edges) + { + cancellationToken.ThrowIfCancellationRequested(); + + var outer = GetOuterArchive(edge.SegmentIdentifier); + if (!builders.TryGetValue(outer, out var builder)) + { + builder = new JniSummaryBuilder(); + builders[outer] = builder; + } + + builder.EdgeCount++; + builder.Reasons.Add(edge.Reason.ToString()); + + if (edge.Reason == JavaJniReason.NativeMethod) + { + builder.NativeMethodCount++; + } + + if (edge.Reason is JavaJniReason.SystemLoad + or JavaJniReason.SystemLoadLibrary + or JavaJniReason.RuntimeLoad + or JavaJniReason.RuntimeLoadLibrary) + { + builder.LoadCallCount++; + } + + if (!string.IsNullOrWhiteSpace(edge.TargetLibrary)) + { + builder.TargetLibraries.Add(edge.TargetLibrary.Trim()); + } + } + + foreach (var warning in analysis.Warnings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var outer = GetOuterArchive(warning.SegmentIdentifier); + if (!builders.TryGetValue(outer, out var builder)) + { + builder = new JniSummaryBuilder(); + builders[outer] = builder; + } + + builder.WarningCount++; + } + + return builders.ToDictionary( + static pair => pair.Key, + static pair => pair.Value.Build(MaxTargetLibraries), + StringComparer.Ordinal); + } + + private static string GetOuterArchive(string segmentIdentifier) + { + if (string.IsNullOrWhiteSpace(segmentIdentifier)) + { + return "."; + } + + var index = segmentIdentifier.IndexOf('!'); + return index < 0 ? segmentIdentifier : segmentIdentifier[..index]; + } + + private sealed class JniSummaryBuilder + { + public int EdgeCount { get; set; } + + public int WarningCount { get; set; } + + public int NativeMethodCount { get; set; } + + public int LoadCallCount { get; set; } + + public SortedSet Reasons { get; } = new(StringComparer.Ordinal); + + public SortedSet TargetLibraries { get; } = new(StringComparer.Ordinal); + + public JniBytecodeSummary Build(int maxTargetLibraries) + { + var targetLibraries = TargetLibraries.Take(Math.Max(0, maxTargetLibraries)).ToArray(); + return new JniBytecodeSummary( + EdgeCount, + WarningCount, + NativeMethodCount, + LoadCallCount, + Reasons.ToArray(), + targetLibraries); + } + } + private static VersionConflictAnalysis BuildConflictAnalysis(JavaLockData lockData) { if (!lockData.HasEntries) @@ -100,11 +227,14 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer JavaLockData lockData, HashSet matchedLocks, bool hasLockEntries, + JniBytecodeSummary? jniBytecodeSummary, CancellationToken cancellationToken) { ManifestMetadata? manifestMetadata = null; OsgiBundleInfo? osgiInfo = null; + var usedByEntrypoint = context.UsageHints.IsPathUsed(archive.AbsolutePath); + if (archive.TryGetEntry("META-INF/MANIFEST.MF", out var manifestEntry)) { var parseResult = await ParseManifestWithOsgiAsync(archive, manifestEntry, cancellationToken).ConfigureAwait(false); @@ -113,11 +243,24 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer } var frameworkConfig = ScanFrameworkConfigs(archive, cancellationToken); - var jniHints = ScanJniHints(archive, cancellationToken); + var jniHints = ScanJniHints(archive, jniBytecodeSummary, cancellationToken); // E1: Detect shaded JARs var shadingResult = await ShadedJarDetector.AnalyzeAsync(archive.AbsolutePath, cancellationToken).ConfigureAwait(false); + // Task 403-001: Scan embedded libraries inside fat archives. + var embeddedScan = await ProcessEmbeddedLibrariesAsync( + archive, + Id, + writer, + lockData, + matchedLocks, + hasLockEntries, + usedByEntrypoint, + cancellationToken).ConfigureAwait(false); + + var emittedFromPomProperties = false; + foreach (var entry in archive.Entries) { cancellationToken.ThrowIfCancellationRequested(); @@ -138,7 +281,10 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer continue; } + emittedFromPomProperties = true; + var metadata = CreateInstalledMetadata(artifact, archive, manifestMetadata, osgiInfo, shadingResult); + AppendEmbeddedScanMetadata(metadata, embeddedScan); if (lockData.TryGet(artifact.GroupId, artifact.ArtifactId, artifact.Version, out var lockEntry)) { @@ -162,7 +308,7 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer var evidence = new List { - new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(archive, entry.OriginalPath), null, artifact.PomSha256), + new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(archive, entry.OriginalPath), null, artifact.EvidenceSha256), }; if (manifestMetadata is not null) @@ -173,8 +319,6 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer evidence.AddRange(frameworkConfig.Evidence); evidence.AddRange(jniHints.Evidence); - var usedByEntrypoint = context.UsageHints.IsPathUsed(archive.AbsolutePath); - writer.AddFromPurl( analyzerId: Id, purl: artifact.Purl, @@ -185,6 +329,188 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer evidence: evidence, usedByEntrypoint: usedByEntrypoint); } + + if (!emittedFromPomProperties) + { + await ProcessPomXmlFallbackAsync( + archive, + manifestMetadata, + osgiInfo, + shadingResult, + embeddedScan, + frameworkConfig, + jniHints, + writer, + lockData, + matchedLocks, + hasLockEntries, + usedByEntrypoint, + cancellationToken).ConfigureAwait(false); + } + } + + private async ValueTask ProcessPomXmlFallbackAsync( + JavaArchive archive, + ManifestMetadata? manifestMetadata, + OsgiBundleInfo? osgiInfo, + ShadingAnalysis? shadingResult, + EmbeddedScanSummary embeddedScan, + FrameworkConfigSummary frameworkConfig, + JniHintSummary jniHints, + LanguageComponentWriter writer, + JavaLockData lockData, + HashSet matchedLocks, + bool hasLockEntries, + bool usedByEntrypoint, + CancellationToken cancellationToken) + { + const long MaxPomXmlBytes = 1_000_000; + + var pomXmlEntries = archive.Entries + .Where(static entry => IsPomXmlEntry(entry.EffectivePath)) + .OrderBy(static entry => entry.EffectivePath, StringComparer.Ordinal) + .ToArray(); + + if (pomXmlEntries.Length == 0) + { + return; + } + + foreach (var pomXmlEntry in pomXmlEntries) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (pomXmlEntry.Length <= 0 || pomXmlEntry.Length > MaxPomXmlBytes) + { + continue; + } + + PomXmlParseResult? pomXml; + try + { + await using var stream = archive.OpenEntry(pomXmlEntry); + pomXml = TryParsePomXml(stream, cancellationToken); + } + catch (IOException) + { + continue; + } + catch (InvalidDataException) + { + continue; + } + + if (pomXml is null) + { + continue; + } + + var resolvedArtifact = TryCreateMavenArtifactFromPomXml(pomXml); + if (resolvedArtifact is not null) + { + var metadata = CreateInstalledMetadata(resolvedArtifact, archive, manifestMetadata, osgiInfo, shadingResult); + AppendEmbeddedScanMetadata(metadata, embeddedScan); + + if (lockData.TryGet(resolvedArtifact.GroupId, resolvedArtifact.ArtifactId, resolvedArtifact.Version, out var lockEntry)) + { + matchedLocks.Add(lockEntry!.Key); + AppendLockMetadata(metadata, lockEntry); + } + else if (hasLockEntries) + { + AddMetadata(metadata, "lockMissing", "true"); + } + + foreach (var hint in frameworkConfig.Metadata) + { + AddMetadata(metadata, hint.Key, hint.Value); + } + + foreach (var hint in jniHints.Metadata) + { + AddMetadata(metadata, hint.Key, hint.Value); + } + + var evidence = new List + { + new(LanguageEvidenceKind.File, "pom.xml", BuildLocator(archive, pomXmlEntry.OriginalPath), null, pomXml.EvidenceSha256), + }; + + if (manifestMetadata is not null) + { + evidence.Add(manifestMetadata.CreateEvidence(archive)); + } + + evidence.AddRange(frameworkConfig.Evidence); + evidence.AddRange(jniHints.Evidence); + + writer.AddFromPurl( + analyzerId: Id, + purl: resolvedArtifact.Purl, + name: resolvedArtifact.ArtifactId, + version: resolvedArtifact.Version, + type: "maven", + metadata: SortMetadata(metadata), + evidence: evidence, + usedByEntrypoint: usedByEntrypoint); + + continue; + } + + var unresolvedMetadata = new List>(16); + AddMetadata(unresolvedMetadata, "unresolvedCoordinates", "true"); + AddMetadata(unresolvedMetadata, "jarPath", NormalizeArchivePath(archive.RelativePath), allowEmpty: true); + AddMetadata(unresolvedMetadata, "groupId", pomXml.GroupId); + AddMetadata(unresolvedMetadata, "artifactId", pomXml.ArtifactId); + AddMetadata(unresolvedMetadata, "packaging", pomXml.Packaging); + AddMetadata(unresolvedMetadata, "displayName", pomXml.Name); + + manifestMetadata?.ApplyMetadata(unresolvedMetadata); + AppendEmbeddedScanMetadata(unresolvedMetadata, embeddedScan); + + foreach (var hint in frameworkConfig.Metadata) + { + AddMetadata(unresolvedMetadata, hint.Key, hint.Value); + } + + foreach (var hint in jniHints.Metadata) + { + AddMetadata(unresolvedMetadata, hint.Key, hint.Value); + } + + var unresolvedEvidence = new List + { + new(LanguageEvidenceKind.File, "pom.xml", BuildLocator(archive, pomXmlEntry.OriginalPath), null, pomXml.EvidenceSha256), + }; + + if (manifestMetadata is not null) + { + unresolvedEvidence.Add(manifestMetadata.CreateEvidence(archive)); + } + + unresolvedEvidence.AddRange(frameworkConfig.Evidence); + unresolvedEvidence.AddRange(jniHints.Evidence); + + var componentName = + pomXml.Name + ?? manifestMetadata?.ImplementationTitle + ?? pomXml.ArtifactId + ?? Path.GetFileNameWithoutExtension(archive.RelativePath); + var explicitKeyName = pomXml.ArtifactId ?? componentName; + var originLocator = BuildLocator(archive, pomXmlEntry.OriginalPath); + var componentKey = LanguageExplicitKey.Create(Id, "maven", explicitKeyName, pomXml.EvidenceSha256, originLocator); + + writer.AddFromExplicitKey( + analyzerId: Id, + componentKey: componentKey, + purl: null, + name: componentName, + version: null, + type: "maven", + metadata: SortMetadata(unresolvedMetadata), + evidence: unresolvedEvidence, + usedByEntrypoint: usedByEntrypoint); + } } private static string BuildLocator(JavaArchive archive, string entryPath) @@ -292,7 +618,7 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer return new FrameworkConfigSummary(flattened, evidence); } - private static JniHintSummary ScanJniHints(JavaArchive archive, CancellationToken cancellationToken) + private static JniHintSummary ScanJniHints(JavaArchive archive, JniBytecodeSummary? jniBytecodeSummary, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(archive); @@ -315,10 +641,27 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer { AddHint(metadata, evidence, "jni.graalConfig", locator, locator, "jni-graal"); } + } - if (IsClassFile(path) && entry.Length is > 0 and < 1_000_000) + if (jniBytecodeSummary is not null && jniBytecodeSummary.EdgeCount > 0) + { + AddHintValue(metadata, "jni.edgeCount", jniBytecodeSummary.EdgeCount.ToString(CultureInfo.InvariantCulture)); + AddHintValue(metadata, "jni.nativeMethodCount", jniBytecodeSummary.NativeMethodCount.ToString(CultureInfo.InvariantCulture)); + AddHintValue(metadata, "jni.loadCallCount", jniBytecodeSummary.LoadCallCount.ToString(CultureInfo.InvariantCulture)); + + if (jniBytecodeSummary.WarningCount > 0) { - TryScanClassForLoadCalls(archive, entry, locator, metadata, evidence, cancellationToken); + AddHintValue(metadata, "jni.warningCount", jniBytecodeSummary.WarningCount.ToString(CultureInfo.InvariantCulture)); + } + + foreach (var reason in jniBytecodeSummary.Reasons) + { + AddHintValue(metadata, "jni.reasons", reason); + } + + foreach (var lib in jniBytecodeSummary.TargetLibraries) + { + AddHintValue(metadata, "jni.targetLibraries", lib); } } @@ -330,76 +673,6 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer return new JniHintSummary(flattened, evidence); } - private static void TryScanClassForLoadCalls( - JavaArchive archive, - JavaArchiveEntry entry, - string locator, - IDictionary> metadata, - ICollection evidence, - CancellationToken cancellationToken) - { - try - { - using var stream = archive.OpenEntry(entry); - using var buffer = new MemoryStream(); - stream.CopyTo(buffer); - var bytes = buffer.ToArray(); - - if (ContainsAscii(bytes, "System.loadLibrary")) - { - AddHint(metadata, evidence, "jni.loadCalls", locator, locator, "jni-load"); - } - else if (ContainsAscii(bytes, "System.load")) - { - AddHint(metadata, evidence, "jni.loadCalls", locator, locator, "jni-load"); - } - } - catch - { - // best effort; skip unreadable class entries - } - } - - private static bool ContainsAscii(byte[] buffer, string ascii) - { - if (buffer.Length == 0 || string.IsNullOrEmpty(ascii)) - { - return false; - } - - var needle = Encoding.ASCII.GetBytes(ascii); - return SpanSearch(buffer, needle) >= 0; - } - - private static int SpanSearch(byte[] haystack, byte[] needle) - { - if (needle.Length == 0 || haystack.Length < needle.Length) - { - return -1; - } - - var lastStart = haystack.Length - needle.Length; - for (var i = 0; i <= lastStart; i++) - { - var matched = true; - for (var j = 0; j < needle.Length; j++) - { - if (haystack[i + j] != needle[j]) - { - matched = false; - break; - } - } - - if (matched) - { - return i; - } - } - - return -1; - } - private static void AddHint( IDictionary> metadata, ICollection evidence, @@ -424,6 +697,25 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer null)); } + private static void AddHintValue( + IDictionary> metadata, + string key, + string value) + { + if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(value)) + { + return; + } + + if (!metadata.TryGetValue(key, out var items)) + { + items = new SortedSet(StringComparer.Ordinal); + metadata[key] = items; + } + + items.Add(value.Trim()); + } + private static void AddConfigHint( IDictionary> metadata, ICollection evidence, @@ -525,6 +817,19 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer || extension.Equals(".jnilib", StringComparison.OrdinalIgnoreCase); } + private static bool IsEmbeddedLibraryJar(string path) + { + if (!path.EndsWith(".jar", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return path.StartsWith("BOOT-INF/lib/", StringComparison.OrdinalIgnoreCase) + || path.StartsWith("WEB-INF/lib/", StringComparison.OrdinalIgnoreCase) + || path.StartsWith("APP-INF/lib/", StringComparison.OrdinalIgnoreCase) + || path.StartsWith("lib/", StringComparison.OrdinalIgnoreCase); + } + private static bool IsClassFile(string path) => path.EndsWith(".class", StringComparison.OrdinalIgnoreCase); @@ -532,6 +837,10 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer => entryName.StartsWith("META-INF/maven/", StringComparison.OrdinalIgnoreCase) && entryName.EndsWith("/pom.properties", StringComparison.OrdinalIgnoreCase); + private static bool IsPomXmlEntry(string entryName) + => entryName.StartsWith("META-INF/maven/", StringComparison.OrdinalIgnoreCase) + && entryName.EndsWith("/pom.xml", StringComparison.OrdinalIgnoreCase); + private static bool IsManifestEntry(string entryName) => string.Equals(entryName, "META-INF/MANIFEST.MF", StringComparison.OrdinalIgnoreCase); @@ -618,9 +927,111 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer Packaging: packaging?.Trim(), Name: name?.Trim(), Purl: purl, - PomSha256: pomSha); + EvidenceSha256: pomSha); } + private sealed record PomXmlParseResult( + string? GroupId, + string? ArtifactId, + string? Version, + string? Packaging, + string? Name, + string EvidenceSha256); + + private static PomXmlParseResult? TryParsePomXml(Stream stream, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + cancellationToken.ThrowIfCancellationRequested(); + + if (buffer.Length == 0) + { + return null; + } + + buffer.Position = 0; + var sha256 = Convert.ToHexString(SHA256.HashData(buffer)).ToLowerInvariant(); + buffer.Position = 0; + + try + { + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + XmlResolver = null, + }; + + using var xmlReader = XmlReader.Create(buffer, settings); + var document = XDocument.Load(xmlReader, LoadOptions.None); + var project = document.Root; + if (project is null) + { + return null; + } + + static string? GetValue(XElement? element, string localName) + { + if (element is null) + { + return null; + } + + var value = element.Elements().FirstOrDefault(e => e.Name.LocalName == localName)?.Value; + return string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + } + + var groupId = GetValue(project, "groupId"); + var artifactId = GetValue(project, "artifactId"); + var version = GetValue(project, "version"); + var packaging = GetValue(project, "packaging"); + var name = GetValue(project, "name"); + + var parent = project.Elements().FirstOrDefault(e => e.Name.LocalName == "parent"); + groupId ??= GetValue(parent, "groupId"); + version ??= GetValue(parent, "version"); + + return new PomXmlParseResult(groupId, artifactId, version, packaging, name, sha256); + } + catch (XmlException) + { + return null; + } + } + + private static MavenArtifact? TryCreateMavenArtifactFromPomXml(PomXmlParseResult pomXml) + { + ArgumentNullException.ThrowIfNull(pomXml); + + if (string.IsNullOrWhiteSpace(pomXml.GroupId) + || string.IsNullOrWhiteSpace(pomXml.ArtifactId) + || string.IsNullOrWhiteSpace(pomXml.Version)) + { + return null; + } + + if (ContainsPlaceholder(pomXml.GroupId) || ContainsPlaceholder(pomXml.ArtifactId) || ContainsPlaceholder(pomXml.Version)) + { + return null; + } + + var packaging = string.IsNullOrWhiteSpace(pomXml.Packaging) ? "jar" : pomXml.Packaging; + var purl = BuildPurl(pomXml.GroupId, pomXml.ArtifactId, pomXml.Version, packaging); + + return new MavenArtifact( + GroupId: pomXml.GroupId, + ArtifactId: pomXml.ArtifactId, + Version: pomXml.Version, + Packaging: packaging, + Name: pomXml.Name, + Purl: purl, + EvidenceSha256: pomXml.EvidenceSha256); + } + + private static bool ContainsPlaceholder(string? value) + => value is not null && value.Contains("${", StringComparison.Ordinal); + private static async ValueTask ParseManifestWithOsgiAsync(JavaArchive archive, JavaArchiveEntry entry, CancellationToken cancellationToken) { await using var entryStream = archive.OpenEntry(entry); @@ -651,6 +1062,54 @@ public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer return new ManifestParseResult(manifestMetadata, osgiInfo); } + private static ManifestMetadata? TryParseManifestMetadata(ZipArchive archive, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(archive); + + ZipArchiveEntry? manifestEntry = null; + foreach (var entry in archive.Entries) + { + cancellationToken.ThrowIfCancellationRequested(); + + var normalized = JavaZipEntryUtilities.NormalizeEntryName(entry.FullName); + if (IsManifestEntry(normalized)) + { + manifestEntry = entry; + break; + } + } + + if (manifestEntry is null) + { + return null; + } + + try + { + using var stream = manifestEntry.Open(); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); + var manifestContent = reader.ReadToEnd(); + cancellationToken.ThrowIfCancellationRequested(); + + var manifestDict = OsgiBundleParser.ParseManifest(manifestContent); + manifestDict.TryGetValue("Implementation-Title", out var title); + manifestDict.TryGetValue("Implementation-Version", out var version); + manifestDict.TryGetValue("Implementation-Vendor", out var vendor); + + return (title is null && version is null && vendor is null) + ? null + : new ManifestMetadata(title, version, vendor); + } + catch (IOException) + { + return null; + } + catch (InvalidDataException) + { + return null; + } + } + private sealed record ManifestParseResult(ManifestMetadata? Manifest, OsgiBundleInfo? OsgiInfo); internal sealed record FrameworkConfigSummary( @@ -688,7 +1147,7 @@ internal sealed record JniHintSummary( string? Packaging, string? Name, string Purl, - string PomSha256); + string EvidenceSha256); private sealed record ManifestMetadata(string? ImplementationTitle, string? ImplementationVersion, string? ImplementationVendor) { @@ -795,6 +1254,430 @@ internal sealed record JniHintSummary( return metadata; } + private static List> CreateEmbeddedMetadata( + MavenArtifact artifact, + string containerJarPath, + string embeddedEntryPath, + string embeddedJarLocator) + { + var metadata = new List>(16); + + AddMetadata(metadata, "groupId", artifact.GroupId); + AddMetadata(metadata, "artifactId", artifact.ArtifactId); + AddMetadata(metadata, "jarPath", embeddedJarLocator, allowEmpty: true); + AddMetadata(metadata, "packaging", artifact.Packaging); + AddMetadata(metadata, "displayName", artifact.Name); + + AddMetadata(metadata, "embedded", "true"); + AddMetadata(metadata, "embedded.containerJarPath", containerJarPath, allowEmpty: true); + AddMetadata(metadata, "embedded.entryPath", embeddedEntryPath, allowEmpty: true); + + return metadata; + } + + private sealed record EmbeddedScanSummary( + int CandidateJars, + int ScannedJars, + int SkippedJars, + int EmittedComponents, + IReadOnlyList SkipReasons); + + private static void AppendEmbeddedScanMetadata( + ICollection> metadata, + EmbeddedScanSummary embeddedScan) + { + ArgumentNullException.ThrowIfNull(metadata); + ArgumentNullException.ThrowIfNull(embeddedScan); + + if (embeddedScan.CandidateJars == 0 && embeddedScan.SkippedJars == 0 && embeddedScan.EmittedComponents == 0) + { + return; + } + + AddMetadata(metadata, "embeddedScan.candidateJars", embeddedScan.CandidateJars.ToString(CultureInfo.InvariantCulture), allowEmpty: true); + AddMetadata(metadata, "embeddedScan.scannedJars", embeddedScan.ScannedJars.ToString(CultureInfo.InvariantCulture), allowEmpty: true); + AddMetadata(metadata, "embeddedScan.emittedComponents", embeddedScan.EmittedComponents.ToString(CultureInfo.InvariantCulture), allowEmpty: true); + + if (embeddedScan.SkippedJars > 0) + { + AddMetadata(metadata, "embeddedScanSkipped", "true"); + AddMetadata(metadata, "embeddedScan.skippedJars", embeddedScan.SkippedJars.ToString(CultureInfo.InvariantCulture), allowEmpty: true); + if (embeddedScan.SkipReasons.Count > 0) + { + AddMetadata(metadata, "embeddedScanSkipReasons", string.Join(",", embeddedScan.SkipReasons), allowEmpty: true); + } + } + } + + private static async ValueTask ProcessEmbeddedLibrariesAsync( + JavaArchive archive, + string analyzerId, + LanguageComponentWriter writer, + JavaLockData lockData, + HashSet matchedLocks, + bool hasLockEntries, + bool usedByEntrypoint, + CancellationToken cancellationToken) + { + const int MaxEmbeddedJarsPerArchive = 256; + const long MaxEmbeddedJarBytes = 25L * 1024 * 1024; + const int MaxSkipReasons = 8; + + ArgumentNullException.ThrowIfNull(archive); + ArgumentException.ThrowIfNullOrWhiteSpace(analyzerId); + ArgumentNullException.ThrowIfNull(writer); + ArgumentNullException.ThrowIfNull(lockData); + ArgumentNullException.ThrowIfNull(matchedLocks); + + var embeddedJars = archive.Entries + .Where(static entry => IsEmbeddedLibraryJar(entry.EffectivePath)) + .OrderBy(static entry => entry.EffectivePath, StringComparer.Ordinal) + .ToArray(); + + if (embeddedJars.Length == 0) + { + return new EmbeddedScanSummary(0, 0, 0, 0, Array.Empty()); + } + + var containerJarPath = NormalizeArchivePath(archive.RelativePath); + + var candidateJars = embeddedJars.Length; + var scannedJars = 0; + var skippedJars = 0; + var emittedComponents = 0; + var skipReasons = new SortedSet(StringComparer.Ordinal); + + for (var i = 0; i < embeddedJars.Length; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (i >= MaxEmbeddedJarsPerArchive) + { + skippedJars += embeddedJars.Length - i; + skipReasons.Add("max-embedded-jars"); + break; + } + + var embeddedJarEntry = embeddedJars[i]; + var embeddedEntryPath = NormalizeEntry(embeddedJarEntry.EffectivePath); + var embeddedJarLocator = BuildLocator(archive, embeddedJarEntry.OriginalPath); + + if (embeddedJarEntry.Length <= 0) + { + skippedJars++; + skipReasons.Add("embedded-jar-empty"); + continue; + } + + if (embeddedJarEntry.Length > MaxEmbeddedJarBytes) + { + skippedJars++; + skipReasons.Add("embedded-jar-too-large"); + continue; + } + + try + { + using var embeddedStream = archive.OpenEntry(embeddedJarEntry); + using var buffer = new MemoryStream(); + await embeddedStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + buffer.Position = 0; + + using var zip = new ZipArchive(buffer, ZipArchiveMode.Read, leaveOpen: true); + scannedJars++; + + var emittedFromPomProperties = false; + + foreach (var zipEntry in zip.Entries.OrderBy(static entry => entry.FullName, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var normalizedName = JavaZipEntryUtilities.NormalizeEntryName(zipEntry.FullName); + if (!IsPomPropertiesEntry(normalizedName)) + { + continue; + } + + MavenArtifact? artifact; + try + { + using var pomStream = zipEntry.Open(); + artifact = TryParsePomProperties(pomStream, cancellationToken); + } + catch (InvalidDataException) + { + continue; + } + catch (IOException) + { + continue; + } + + if (artifact is null) + { + continue; + } + + emittedFromPomProperties = true; + emittedComponents++; + + var metadata = CreateEmbeddedMetadata(artifact, containerJarPath, embeddedEntryPath, embeddedJarLocator); + + if (lockData.TryGet(artifact.GroupId, artifact.ArtifactId, artifact.Version, out var lockEntry)) + { + matchedLocks.Add(lockEntry!.Key); + AppendLockMetadata(metadata, lockEntry); + } + else if (hasLockEntries) + { + AddMetadata(metadata, "lockMissing", "true"); + } + + var locator = string.Concat(embeddedJarLocator, "!", NormalizeEntry(normalizedName)); + var evidence = new[] + { + new LanguageComponentEvidence( + LanguageEvidenceKind.File, + "pom.properties", + locator, + null, + artifact.EvidenceSha256), + }; + + writer.AddFromPurl( + analyzerId: analyzerId, + purl: artifact.Purl, + name: artifact.ArtifactId, + version: artifact.Version, + type: "maven", + metadata: SortMetadata(metadata), + evidence: evidence, + usedByEntrypoint: usedByEntrypoint); + } + + if (!emittedFromPomProperties) + { + const long MaxPomXmlBytes = 1_000_000; + var embeddedManifestMetadata = TryParseManifestMetadata(zip, cancellationToken); + + foreach (var zipEntry in zip.Entries.OrderBy(static entry => entry.FullName, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var normalizedName = JavaZipEntryUtilities.NormalizeEntryName(zipEntry.FullName); + if (!IsPomXmlEntry(normalizedName)) + { + continue; + } + + if (zipEntry.Length <= 0 || zipEntry.Length > MaxPomXmlBytes) + { + continue; + } + + PomXmlParseResult? pomXml; + try + { + using var pomStream = zipEntry.Open(); + pomXml = TryParsePomXml(pomStream, cancellationToken); + } + catch (InvalidDataException) + { + continue; + } + catch (IOException) + { + continue; + } + + if (pomXml is null) + { + continue; + } + + var resolvedArtifact = TryCreateMavenArtifactFromPomXml(pomXml); + if (resolvedArtifact is not null) + { + emittedComponents++; + + var metadata = CreateEmbeddedMetadata(resolvedArtifact, containerJarPath, embeddedEntryPath, embeddedJarLocator); + + if (lockData.TryGet(resolvedArtifact.GroupId, resolvedArtifact.ArtifactId, resolvedArtifact.Version, out var lockEntry)) + { + matchedLocks.Add(lockEntry!.Key); + AppendLockMetadata(metadata, lockEntry); + } + else if (hasLockEntries) + { + AddMetadata(metadata, "lockMissing", "true"); + } + + var locator = string.Concat(embeddedJarLocator, "!", NormalizeEntry(normalizedName)); + var evidence = new[] + { + new LanguageComponentEvidence( + LanguageEvidenceKind.File, + "pom.xml", + locator, + null, + pomXml.EvidenceSha256), + }; + + writer.AddFromPurl( + analyzerId: analyzerId, + purl: resolvedArtifact.Purl, + name: resolvedArtifact.ArtifactId, + version: resolvedArtifact.Version, + type: "maven", + metadata: SortMetadata(metadata), + evidence: evidence, + usedByEntrypoint: usedByEntrypoint); + + continue; + } + + emittedComponents++; + + var unresolvedMetadata = new List>(16); + AddMetadata(unresolvedMetadata, "unresolvedCoordinates", "true"); + AddMetadata(unresolvedMetadata, "jarPath", embeddedJarLocator, allowEmpty: true); + AddMetadata(unresolvedMetadata, "groupId", pomXml.GroupId); + AddMetadata(unresolvedMetadata, "artifactId", pomXml.ArtifactId); + AddMetadata(unresolvedMetadata, "packaging", pomXml.Packaging); + AddMetadata(unresolvedMetadata, "displayName", pomXml.Name); + AddMetadata(unresolvedMetadata, "embedded", "true"); + AddMetadata(unresolvedMetadata, "embedded.containerJarPath", containerJarPath, allowEmpty: true); + AddMetadata(unresolvedMetadata, "embedded.entryPath", embeddedEntryPath, allowEmpty: true); + + embeddedManifestMetadata?.ApplyMetadata(unresolvedMetadata); + + var locatorUnresolved = string.Concat(embeddedJarLocator, "!", NormalizeEntry(normalizedName)); + var unresolvedEvidence = new[] + { + new LanguageComponentEvidence( + LanguageEvidenceKind.File, + "pom.xml", + locatorUnresolved, + null, + pomXml.EvidenceSha256), + }; + + var componentName = + pomXml.Name + ?? embeddedManifestMetadata?.ImplementationTitle + ?? pomXml.ArtifactId + ?? Path.GetFileNameWithoutExtension(embeddedEntryPath); + + var explicitKeyName = pomXml.ArtifactId ?? componentName; + var componentKey = LanguageExplicitKey.Create(analyzerId, "maven", explicitKeyName, pomXml.EvidenceSha256, locatorUnresolved); + + writer.AddFromExplicitKey( + analyzerId: analyzerId, + componentKey: componentKey, + purl: null, + name: componentName, + version: null, + type: "maven", + metadata: SortMetadata(unresolvedMetadata), + evidence: unresolvedEvidence, + usedByEntrypoint: usedByEntrypoint); + } + } + } + catch (InvalidDataException) + { + skippedJars++; + skipReasons.Add("embedded-jar-invalid-zip"); + } + catch (IOException) + { + skippedJars++; + skipReasons.Add("embedded-jar-io-error"); + } + + if (skipReasons.Count >= MaxSkipReasons) + { + break; + } + } + + return new EmbeddedScanSummary( + CandidateJars: candidateJars, + ScannedJars: scannedJars, + SkippedJars: skippedJars, + EmittedComponents: emittedComponents, + SkipReasons: skipReasons.ToArray()); + } + + private static MavenArtifact? TryParsePomProperties(Stream stream, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(stream); + + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + buffer.Position = 0; + + using var reader = new StreamReader(buffer, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: true); + var properties = new Dictionary(StringComparer.OrdinalIgnoreCase); + + while (reader.ReadLine() is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + line = line.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + { + continue; + } + + var separatorIndex = line.IndexOf('='); + if (separatorIndex <= 0) + { + continue; + } + + var key = line[..separatorIndex].Trim(); + var value = line[(separatorIndex + 1)..].Trim(); + if (key.Length == 0) + { + continue; + } + + properties[key] = value; + } + + buffer.Position = 0; + var sha256 = Convert.ToHexString(SHA256.HashData(buffer)).ToLowerInvariant(); + + if (!properties.TryGetValue("groupId", out var groupId) || string.IsNullOrWhiteSpace(groupId)) + { + return null; + } + + if (!properties.TryGetValue("artifactId", out var artifactId) || string.IsNullOrWhiteSpace(artifactId)) + { + return null; + } + + if (!properties.TryGetValue("version", out var version) || string.IsNullOrWhiteSpace(version)) + { + return null; + } + + var packaging = properties.TryGetValue("packaging", out var packagingValue) ? packagingValue : "jar"; + var name = properties.TryGetValue("name", out var nameValue) ? nameValue : null; + var purl = BuildPurl(groupId, artifactId, version, packaging); + + return new MavenArtifact( + GroupId: groupId.Trim(), + ArtifactId: artifactId.Trim(), + Version: version.Trim(), + Packaging: packaging?.Trim(), + Name: name?.Trim(), + Purl: purl, + EvidenceSha256: sha256); + } + private static IReadOnlyList> CreateDeclaredMetadata( JavaLockEntry entry, VersionConflictAnalysis conflictAnalysis) diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeDependencyIndex.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeDependencyIndex.cs index f6cf2cf53..0691a257c 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeDependencyIndex.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeDependencyIndex.cs @@ -30,6 +30,11 @@ internal sealed class NodeDependencyIndex /// A dependency index with all declared dependencies and their scopes. public static NodeDependencyIndex Create(string rootPath) { + if (string.IsNullOrWhiteSpace(rootPath)) + { + return Empty; + } + var packageJsonPath = Path.Combine(rootPath, "package.json"); if (!File.Exists(packageJsonPath)) { @@ -40,7 +45,75 @@ internal sealed class NodeDependencyIndex { using var stream = File.OpenRead(packageJsonPath); using var document = JsonDocument.Parse(stream); - return CreateFromJson(document.RootElement); + + var rootDeclarations = BuildDeclarationsFromJson(document.RootElement); + if (rootDeclarations.Count == 0) + { + rootDeclarations = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + var workspaceBest = new Dictionary(StringComparer.OrdinalIgnoreCase); + var workspaceIndex = NodeWorkspaceIndex.Create(rootPath); + foreach (var workspaceRelative in workspaceIndex.GetMembers()) + { + var workspacePackageJsonPath = Path.Combine(rootPath, workspaceRelative.Replace('/', Path.DirectorySeparatorChar), "package.json"); + if (!File.Exists(workspacePackageJsonPath)) + { + continue; + } + + try + { + using var workspaceStream = File.OpenRead(workspacePackageJsonPath); + using var workspaceDocument = JsonDocument.Parse(workspaceStream); + var workspaceDeclarations = BuildDeclarationsFromJson(workspaceDocument.RootElement); + foreach (var declaration in workspaceDeclarations.Values) + { + if (rootDeclarations.ContainsKey(declaration.Name)) + { + continue; + } + + if (workspaceBest.TryGetValue(declaration.Name, out var existing)) + { + if (GetScopePriority(declaration.Scope) < GetScopePriority(existing.Scope)) + { + workspaceBest[declaration.Name] = declaration; + } + + continue; + } + + workspaceBest[declaration.Name] = declaration; + } + } + catch (IOException) + { + continue; + } + catch (JsonException) + { + continue; + } + } + + var merged = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in rootDeclarations) + { + merged[pair.Key] = pair.Value; + } + + foreach (var pair in workspaceBest) + { + merged.TryAdd(pair.Key, pair.Value); + } + + if (merged.Count == 0) + { + return Empty; + } + + return new NodeDependencyIndex(merged); } catch (IOException) { @@ -59,19 +132,31 @@ internal sealed class NodeDependencyIndex /// A dependency index with all declared dependencies and their scopes. public static NodeDependencyIndex CreateFromJson(JsonElement root) { - var declarations = new Dictionary(StringComparer.OrdinalIgnoreCase); + var declarations = BuildDeclarationsFromJson(root); + return declarations.Count == 0 ? Empty : new NodeDependencyIndex(declarations); + } - ParseDependencySection(root, "dependencies", NodeDependencyScope.Production, declarations); - ParseDependencySection(root, "devDependencies", NodeDependencyScope.Development, declarations); - ParseDependencySection(root, "peerDependencies", NodeDependencyScope.Peer, declarations); - ParseDependencySection(root, "optionalDependencies", NodeDependencyScope.Optional, declarations); - - if (declarations.Count == 0) + public static NodeDependencyIndex CreateFromPackageJsonPath(string packageJsonPath) + { + if (string.IsNullOrWhiteSpace(packageJsonPath) || !File.Exists(packageJsonPath)) { return Empty; } - return new NodeDependencyIndex(declarations); + try + { + using var stream = File.OpenRead(packageJsonPath); + using var document = JsonDocument.Parse(stream); + return CreateFromJson(document.RootElement); + } + catch (IOException) + { + return Empty; + } + catch (JsonException) + { + return Empty; + } } /// @@ -159,6 +244,27 @@ internal sealed class NodeDependencyIndex sectionName); } } + + private static Dictionary BuildDeclarationsFromJson(JsonElement root) + { + var declarations = new Dictionary(StringComparer.OrdinalIgnoreCase); + + ParseDependencySection(root, "dependencies", NodeDependencyScope.Production, declarations); + ParseDependencySection(root, "devDependencies", NodeDependencyScope.Development, declarations); + ParseDependencySection(root, "peerDependencies", NodeDependencyScope.Peer, declarations); + ParseDependencySection(root, "optionalDependencies", NodeDependencyScope.Optional, declarations); + + return declarations; + } + + private static int GetScopePriority(NodeDependencyScope scope) => scope switch + { + NodeDependencyScope.Production => 0, + NodeDependencyScope.Development => 1, + NodeDependencyScope.Peer => 2, + NodeDependencyScope.Optional => 3, + _ => 99 + }; } /// diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs index bf6db7d74..b2029c68a 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeImportWalker.cs @@ -11,6 +11,7 @@ namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; internal static class NodeImportWalker { private const int MaxSourceMapBytes = 1_048_576; // 1 MiB safety cap + private const int MaxFallbackBytes = 524_288; // 512 KiB safety cap public static IReadOnlyList AnalyzeImports(string rootPath, string sourcePath, string content) { @@ -29,20 +30,32 @@ internal static class NodeImportWalker private static void AnalyzeInternal(string rootPath, string sourcePath, string content, bool allowSourceMap, List edges) { - Script script; + Program? program; try { var parser = new JavaScriptParser(new ParserOptions { Tolerant = true }); - script = parser.ParseScript(content, sourcePath, strict: false); + program = parser.ParseScript(content, sourcePath, strict: false); } catch (ParserException) { - script = null!; + try + { + var parser = new JavaScriptParser(new ParserOptions { Tolerant = true }); + program = parser.ParseModule(content, sourcePath); + } + catch (ParserException) + { + program = null; + } } - if (script is not null) + if (program is not null) { - Walk(script, sourcePath, edges); + Walk(program, sourcePath, edges); + } + else + { + TryAnalyzeTypeScriptFallback(sourcePath, content, edges); } if (allowSourceMap) @@ -75,6 +88,20 @@ internal static class NodeImportWalker AddEdge(edges, sourcePath, importExprTarget!, "import()", importExprEvidence, importExprConfidence); } break; + + case ExportNamedDeclaration exportNamed when exportNamed.Source is not null: + if (TryGetLiteral(exportNamed.Source, out var exportTarget, out var exportConfidence, out var exportEvidence)) + { + AddEdge(edges, sourcePath, exportTarget!, "export-from", exportEvidence, exportConfidence); + } + break; + + case ExportAllDeclaration exportAll: + if (TryGetLiteral(exportAll.Source, out var exportAllTarget, out var exportAllConfidence, out var exportAllEvidence)) + { + AddEdge(edges, sourcePath, exportAllTarget!, "export-all", exportAllEvidence, exportAllConfidence); + } + break; } foreach (var child in node.ChildNodes) @@ -261,6 +288,49 @@ internal static class NodeImportWalker && call.Arguments.Count == 1; } + private static void TryAnalyzeTypeScriptFallback(string sourcePath, string content, List edges) + { + if (!IsTypeScriptSource(sourcePath)) + { + return; + } + + if (string.IsNullOrWhiteSpace(content)) + { + return; + } + + var input = content.Length > MaxFallbackBytes ? content[..MaxFallbackBytes] : content; + + foreach (Match match in Regex.Matches(input, "\\bimport\\s+(?:type\\s+)?(?:[^;\\n]*?\\s+from\\s+)?[\\\"'](?[^\\\"']+)[\\\"']", RegexOptions.Multiline)) + { + var spec = match.Groups["spec"].Value; + if (string.IsNullOrWhiteSpace(spec)) + { + continue; + } + + AddEdge(edges, sourcePath, spec.Trim(), "import", "ts-regex", "medium"); + } + + foreach (Match match in Regex.Matches(input, "\\bexport\\s+(?:type\\s+)?[^;\\n]*?\\s+from\\s+[\\\"'](?[^\\\"']+)[\\\"']", RegexOptions.Multiline)) + { + var spec = match.Groups["spec"].Value; + if (string.IsNullOrWhiteSpace(spec)) + { + continue; + } + + AddEdge(edges, sourcePath, spec.Trim(), "export-from", "ts-regex", "medium"); + } + } + + private static bool IsTypeScriptSource(string sourcePath) + => sourcePath.EndsWith(".ts", StringComparison.OrdinalIgnoreCase) + || sourcePath.EndsWith(".tsx", StringComparison.OrdinalIgnoreCase) + || sourcePath.EndsWith(".mts", StringComparison.OrdinalIgnoreCase) + || sourcePath.EndsWith(".cts", StringComparison.OrdinalIgnoreCase); + private static string CombineConfidence(string left, string right) { static int Score(string value) => value switch diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs index e2724e5b4..0f483532b 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs @@ -11,22 +11,26 @@ internal sealed class NodeLockData private static readonly NodeLockData Empty = new( new Dictionary(StringComparer.Ordinal), new Dictionary(StringComparer.OrdinalIgnoreCase), + new Dictionary(StringComparer.OrdinalIgnoreCase), Array.Empty(), NodeDependencyIndex.Create(string.Empty)); private readonly Dictionary _byPath; private readonly Dictionary _byName; + private readonly Dictionary _byNameVersion; private readonly IReadOnlyCollection _declared; private readonly NodeDependencyIndex _dependencyIndex; private NodeLockData( Dictionary byPath, Dictionary byName, + Dictionary byNameVersion, IReadOnlyCollection declared, NodeDependencyIndex dependencyIndex) { _byPath = byPath; _byName = byName; + _byNameVersion = byNameVersion; _declared = declared; _dependencyIndex = dependencyIndex; } @@ -42,14 +46,15 @@ internal sealed class NodeLockData { var byPath = new Dictionary(StringComparer.Ordinal); var byName = new Dictionary(StringComparer.OrdinalIgnoreCase); + var byNameVersion = new Dictionary(StringComparer.OrdinalIgnoreCase); var declared = new Dictionary(StringComparer.OrdinalIgnoreCase); // Build dependency index from package.json first var dependencyIndex = NodeDependencyIndex.Create(rootPath); - LoadPackageLockJson(rootPath, byPath, byName, declared, dependencyIndex, cancellationToken); - LoadYarnLock(rootPath, byName, declared, dependencyIndex); - LoadPnpmLock(rootPath, byName, declared, dependencyIndex); + LoadPackageLockJson(rootPath, byPath, byName, byNameVersion, declared, dependencyIndex, cancellationToken); + LoadYarnLock(rootPath, byName, byNameVersion, declared, dependencyIndex); + LoadPnpmLock(rootPath, byName, byNameVersion, declared, dependencyIndex); // Add declared-only entries for packages in package.json but not in any lockfile AddDeclaredOnlyFromPackageJson(declared, dependencyIndex); @@ -66,7 +71,7 @@ internal sealed class NodeLockData .ThenBy(static entry => entry.Locator ?? string.Empty, StringComparer.OrdinalIgnoreCase) .ToArray(); - return ValueTask.FromResult(new NodeLockData(byPath, byName, declaredList, dependencyIndex)); + return ValueTask.FromResult(new NodeLockData(byPath, byName, byNameVersion, declaredList, dependencyIndex)); } /// @@ -110,6 +115,9 @@ internal sealed class NodeLockData } public bool TryGet(string relativePath, string packageName, out NodeLockEntry? entry) + => TryGet(relativePath, packageName, version: null, out entry); + + public bool TryGet(string relativePath, string packageName, string? version, out NodeLockEntry? entry) { var normalizedPath = NormalizeLockPath(relativePath); if (_byPath.TryGetValue(normalizedPath, out var byPathEntry)) @@ -120,8 +128,17 @@ internal sealed class NodeLockData if (!string.IsNullOrEmpty(packageName)) { - var normalizedName = packageName.StartsWith('@') ? packageName : packageName; - if (_byName.TryGetValue(normalizedName, out var byNameEntry)) + if (!string.IsNullOrWhiteSpace(version)) + { + var nameVersionKey = CreateNameVersionKey(packageName, version); + if (_byNameVersion.TryGetValue(nameVersionKey, out var byVersionEntry)) + { + entry = byVersionEntry; + return true; + } + } + + if (_byName.TryGetValue(packageName, out var byNameEntry)) { entry = byNameEntry; return true; @@ -197,6 +214,7 @@ internal sealed class NodeLockData JsonElement dependenciesElement, IDictionary byPath, IDictionary byName, + IDictionary byNameVersion, IDictionary declared, NodeDependencyIndex dependencyIndex) { @@ -210,12 +228,13 @@ internal sealed class NodeLockData { byPath[normalizedPath] = entry; byName[dependency.Name] = entry; + AddNameVersionEntry(byNameVersion, entry); AddDeclaration(declared, entry); } if (depValue.TryGetProperty("dependencies", out var childDependencies) && childDependencies.ValueKind == JsonValueKind.Object) { - TraverseLegacyDependencies(path + "/node_modules", childDependencies, byPath, byName, declared, dependencyIndex); + TraverseLegacyDependencies(path + "/node_modules", childDependencies, byPath, byName, byNameVersion, declared, dependencyIndex); } } } @@ -224,6 +243,7 @@ internal sealed class NodeLockData string rootPath, IDictionary byPath, IDictionary byName, + IDictionary byNameVersion, IDictionary declared, NodeDependencyIndex dependencyIndex, CancellationToken cancellationToken) @@ -260,6 +280,7 @@ internal sealed class NodeLockData if (!string.IsNullOrEmpty(entry.Name)) { byName[entry.Name] = entry; + AddNameVersionEntry(byNameVersion, entry); } AddDeclaration(declared, entry); @@ -267,7 +288,7 @@ internal sealed class NodeLockData } else if (root.TryGetProperty("dependencies", out var dependenciesElement) && dependenciesElement.ValueKind == JsonValueKind.Object) { - TraverseLegacyDependencies("node_modules", dependenciesElement, byPath, byName, declared, dependencyIndex); + TraverseLegacyDependencies("node_modules", dependenciesElement, byPath, byName, byNameVersion, declared, dependencyIndex); } } catch (IOException) @@ -283,6 +304,7 @@ internal sealed class NodeLockData private static void LoadYarnLock( string rootPath, IDictionary byName, + IDictionary byNameVersion, IDictionary declared, NodeDependencyIndex dependencyIndex) { @@ -299,6 +321,7 @@ internal sealed class NodeLockData string? version = null; string? resolved = null; string? integrity = null; + string? checksum = null; void Flush() { @@ -307,6 +330,7 @@ internal sealed class NodeLockData version = null; resolved = null; integrity = null; + checksum = null; return; } @@ -316,6 +340,7 @@ internal sealed class NodeLockData version = null; resolved = null; integrity = null; + checksum = null; return; } @@ -328,12 +353,20 @@ internal sealed class NodeLockData isOptional = foundScope == NodeDependencyScope.Optional; } - var entry = new NodeLockEntry(YarnLockSource, currentName, simpleName, version, resolved, integrity, scope, isOptional); + var effectiveIntegrity = integrity; + if (string.IsNullOrWhiteSpace(effectiveIntegrity) && !string.IsNullOrWhiteSpace(checksum)) + { + effectiveIntegrity = "checksum:" + checksum.Trim(); + } + + var entry = new NodeLockEntry(YarnLockSource, currentName, simpleName, version, resolved, effectiveIntegrity, scope, isOptional); byName[simpleName] = entry; + AddNameVersionEntry(byNameVersion, entry); AddDeclaration(declared, entry); version = null; resolved = null; integrity = null; + checksum = null; } foreach (var line in lines) @@ -353,17 +386,30 @@ internal sealed class NodeLockData continue; } - if (trimmed.StartsWith("version", StringComparison.OrdinalIgnoreCase)) + if (!TryParseYarnField(trimmed, out var key, out var value)) { - version = ExtractQuotedValue(trimmed); + continue; } - else if (trimmed.StartsWith("resolved", StringComparison.OrdinalIgnoreCase)) + + if (key.Equals("version", StringComparison.OrdinalIgnoreCase)) { - resolved = ExtractQuotedValue(trimmed); + version = value; } - else if (trimmed.StartsWith("integrity", StringComparison.OrdinalIgnoreCase)) + else if (key.Equals("resolved", StringComparison.OrdinalIgnoreCase)) { - integrity = ExtractQuotedValue(trimmed); + resolved = value; + } + else if (key.Equals("resolution", StringComparison.OrdinalIgnoreCase)) + { + resolved = value; + } + else if (key.Equals("integrity", StringComparison.OrdinalIgnoreCase)) + { + integrity = value; + } + else if (key.Equals("checksum", StringComparison.OrdinalIgnoreCase)) + { + checksum = value; } } @@ -378,6 +424,7 @@ internal sealed class NodeLockData private static void LoadPnpmLock( string rootPath, IDictionary byName, + IDictionary byNameVersion, IDictionary declared, NodeDependencyIndex dependencyIndex) { @@ -394,11 +441,23 @@ internal sealed class NodeLockData string? version = null; string? resolved = null; string? integrity = null; - var inPackages = false; + string? section = null; void Flush() { - if (string.IsNullOrEmpty(currentPackage) || string.IsNullOrEmpty(integrity)) + if (string.IsNullOrEmpty(currentPackage)) + { + version = null; + resolved = null; + integrity = null; + return; + } + + var effectiveVersion = string.IsNullOrWhiteSpace(version) + ? ExtractVersionFromPnpmKey(currentPackage!) + : version!; + + if (string.IsNullOrWhiteSpace(effectiveVersion)) { version = null; resolved = null; @@ -424,8 +483,12 @@ internal sealed class NodeLockData isOptional = foundScope == NodeDependencyScope.Optional; } - var entry = new NodeLockEntry(PnpmLockSource, currentPackage, name, version, resolved, integrity, scope, isOptional); + var integrityMissing = string.IsNullOrWhiteSpace(integrity); + var integrityReason = integrityMissing ? DetermineIntegrityMissingReason(currentPackage!, resolved, effectiveVersion) : null; + + var entry = new NodeLockEntry(PnpmLockSource, currentPackage, name, effectiveVersion.Trim(), resolved, integrity, scope, isOptional, integrityMissing, integrityReason); byName[name] = entry; + AddNameVersionEntry(byNameVersion, entry); AddDeclaration(declared, entry); version = null; resolved = null; @@ -439,12 +502,30 @@ internal sealed class NodeLockData continue; } - if (!inPackages) + if (!line.StartsWith(' ') && line.EndsWith(':')) + { + Flush(); + currentPackage = null; + version = null; + resolved = null; + integrity = null; + section = null; + } + + if (section is null) { if (line.StartsWith("packages:", StringComparison.Ordinal)) { - inPackages = true; + section = "packages"; + continue; } + + if (line.StartsWith("snapshots:", StringComparison.Ordinal)) + { + section = "snapshots"; + continue; + } + continue; } @@ -471,6 +552,12 @@ internal sealed class NodeLockData if (integrityIndex >= 0) { var integrityValue = trimmed[(integrityIndex + 9)..].Trim(' ', ':', '{', '}', '"'); + var integrityCommaIndex = integrityValue.IndexOf(','); + if (integrityCommaIndex > 0) + { + integrityValue = integrityValue[..integrityCommaIndex].Trim(); + } + integrity = integrityValue; } @@ -478,6 +565,12 @@ internal sealed class NodeLockData if (tarballIndex >= 0) { var tarballValue = trimmed[(tarballIndex + 7)..].Trim(' ', ':', '{', '}', '"'); + var tarballCommaIndex = tarballValue.IndexOf(','); + if (tarballCommaIndex > 0) + { + tarballValue = tarballValue[..tarballCommaIndex].Trim(); + } + resolved = tarballValue; } } @@ -503,6 +596,95 @@ internal sealed class NodeLockData } } + private static bool TryParseYarnField(string trimmed, out string key, out string value) + { + key = string.Empty; + value = string.Empty; + + if (string.IsNullOrWhiteSpace(trimmed)) + { + return false; + } + + var spaceIndex = trimmed.IndexOf(' '); + var colonIndex = trimmed.IndexOf(':'); + if (colonIndex > 0 && (spaceIndex < 0 || colonIndex < spaceIndex)) + { + key = trimmed[..colonIndex].Trim(); + value = TrimYarnScalar(trimmed[(colonIndex + 1)..]); + return key.Length > 0; + } + + if (spaceIndex <= 0) + { + return false; + } + + key = trimmed[..spaceIndex].Trim(); + value = TrimYarnScalar(trimmed[(spaceIndex + 1)..]); + return key.Length > 0; + } + + private static string TrimYarnScalar(string value) + { + value = value.Trim(); + if (value.Length >= 2 && value[0] == '"' && value[^1] == '"') + { + value = value[1..^1]; + } + + return value; + } + + private static string ExtractVersionFromPnpmKey(string key) + { + var parts = key.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0) + { + return string.Empty; + } + + var last = parts[^1]; + var parenIndex = last.IndexOf('('); + if (parenIndex > 0) + { + last = last[..parenIndex]; + } + + return last; + } + + private static string DetermineIntegrityMissingReason(string locator, string? resolved, string version) + { + var candidate = (locator + " " + (resolved ?? string.Empty) + " " + version).ToLowerInvariant(); + if (candidate.Contains("workspace:", StringComparison.Ordinal)) + { + return "workspace"; + } + + if (candidate.Contains("link:", StringComparison.Ordinal)) + { + return "link"; + } + + if (candidate.Contains("file:", StringComparison.Ordinal)) + { + return "file"; + } + + if (candidate.Contains("git", StringComparison.Ordinal)) + { + return "git"; + } + + if (candidate.Contains("directory:", StringComparison.Ordinal)) + { + return "directory"; + } + + return "missing"; + } + private static string? ExtractQuotedValue(string line) { var quoteStart = line.IndexOf('"'); @@ -522,6 +704,11 @@ internal sealed class NodeLockData private static string ExtractPackageNameFromYarnKey(string key) { + if (string.Equals(key, "__metadata", StringComparison.OrdinalIgnoreCase)) + { + return string.Empty; + } + var commaIndex = key.IndexOf(','); var trimmed = commaIndex > 0 ? key[..commaIndex] : key; trimmed = trimmed.Trim('"'); @@ -570,6 +757,25 @@ internal sealed class NodeLockData } } + private static void AddNameVersionEntry(IDictionary byNameVersion, NodeLockEntry entry) + { + if (byNameVersion is null || entry is null) + { + return; + } + + if (string.IsNullOrWhiteSpace(entry.Name) || string.IsNullOrWhiteSpace(entry.Version)) + { + return; + } + + var key = CreateNameVersionKey(entry.Name, entry.Version); + byNameVersion[key] = entry; + } + + private static string CreateNameVersionKey(string name, string version) + => $"{name.Trim()}@{version.Trim()}".ToLowerInvariant(); + private static string NormalizeLockPath(string path) { if (string.IsNullOrWhiteSpace(path)) @@ -595,22 +801,32 @@ internal sealed class NodeLockData return string.Empty; } - if (segments[0] == "node_modules") + var nodeModulesIndex = -1; + for (var i = segments.Length - 1; i >= 0; i--) { - if (segments.Length >= 3 && segments[1].StartsWith('@')) + if (string.Equals(segments[i], "node_modules", StringComparison.Ordinal)) { - return $"{segments[1]}/{segments[2]}"; + nodeModulesIndex = i; + break; + } + } + + if (nodeModulesIndex >= 0) + { + if (nodeModulesIndex + 1 >= segments.Length) + { + return string.Empty; } - return segments.Length >= 2 ? segments[1] : string.Empty; + var candidate = segments[nodeModulesIndex + 1]; + if (candidate.StartsWith('@') && nodeModulesIndex + 2 < segments.Length) + { + return $"{candidate}/{segments[nodeModulesIndex + 2]}"; + } + + return candidate; } - var last = segments[^1]; - if (last.StartsWith('@') && segments.Length >= 2) - { - return $"{segments[^2]}/{last}"; - } - - return last; + return segments[^1]; } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs index ad71024fc..686c31715 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs @@ -19,7 +19,9 @@ internal sealed record NodeLockEntry( string? Resolved, string? Integrity, NodeDependencyScope? Scope = null, - bool IsOptional = false); + bool IsOptional = false, + bool IntegrityMissing = false, + string? IntegrityMissingReason = null); internal static class NodeLockEntryExtensions { diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs index 8af6dc4a8..ff0175cb0 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs @@ -6,6 +6,8 @@ namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; internal sealed class NodePackage { + private readonly Dictionary _extraMetadata = new(StringComparer.Ordinal); + public NodePackage( string name, string version, @@ -212,6 +214,13 @@ internal sealed class NodePackage .ToArray(); } + public void MarkImportScanSkipped(int filesScanned, long bytesScanned) + { + _extraMetadata["importScanSkipped"] = "true"; + _extraMetadata["importScan.filesScanned"] = filesScanned.ToString(CultureInfo.InvariantCulture); + _extraMetadata["importScan.bytesScanned"] = bytesScanned.ToString(CultureInfo.InvariantCulture); + } + public IReadOnlyCollection> CreateMetadata() { var entries = new List>(8) @@ -360,6 +369,14 @@ internal sealed class NodePackage entries.Add(new KeyValuePair("license", License)); } + foreach (var pair in _extraMetadata) + { + if (!string.IsNullOrWhiteSpace(pair.Key) && pair.Value is not null) + { + entries.Add(new KeyValuePair(pair.Key, pair.Value)); + } + } + return entries .OrderBy(static pair => pair.Key, StringComparer.Ordinal) .ToArray(); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs index 84be9047c..fda7cee1e 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs @@ -13,6 +13,26 @@ internal static class NodePackageCollector "__pycache__" }; + private static IReadOnlyDictionary BuildWorkspaceDependencyIndices(string rootPath, NodeWorkspaceIndex workspaceIndex) + { + var comparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var indices = new Dictionary(comparer); + + foreach (var workspaceRelative in workspaceIndex.GetMembers()) + { + var workspaceAbsolute = Path.Combine(rootPath, workspaceRelative.Replace('/', Path.DirectorySeparatorChar)); + var packageJsonPath = Path.Combine(workspaceAbsolute, "package.json"); + if (!File.Exists(packageJsonPath)) + { + continue; + } + + indices[workspaceRelative] = NodeDependencyIndex.CreateFromPackageJsonPath(packageJsonPath); + } + + return indices; + } + public static IReadOnlyList CollectPackages(LanguageAnalyzerContext context, NodeLockData lockData, NodeProjectInput projectInput, CancellationToken cancellationToken) { var packages = new List(); @@ -21,11 +41,12 @@ internal static class NodePackageCollector var rootPackageJson = Path.Combine(context.RootPath, "package.json"); var workspaceIndex = NodeWorkspaceIndex.Create(context.RootPath); + var workspaceDependencyIndices = BuildWorkspaceDependencyIndices(context.RootPath, workspaceIndex); var yarnPnpPresent = projectInput.YarnPnpPresent; if (File.Exists(rootPackageJson)) { - var rootPackage = TryCreatePackage(context, rootPackageJson, string.Empty, lockData, workspaceIndex, yarnPnpPresent, cancellationToken); + var rootPackage = TryCreatePackage(context, rootPackageJson, string.Empty, lockData, workspaceIndex, workspaceDependencyIndices, yarnPnpPresent, cancellationToken); if (rootPackage is not null) { packages.Add(rootPackage); @@ -41,7 +62,7 @@ internal static class NodePackageCollector continue; } - ProcessPackageDirectory(context, workspaceAbsolute, lockData, workspaceIndex, includeNestedNodeModules: false, packages, visited, yarnPnpPresent, cancellationToken); + ProcessPackageDirectory(context, workspaceAbsolute, lockData, workspaceIndex, workspaceDependencyIndices, includeNestedNodeModules: false, packages, visited, yarnPnpPresent, cancellationToken); var workspaceNodeModules = Path.Combine(workspaceAbsolute, "node_modules"); if (Directory.Exists(workspaceNodeModules)) @@ -50,9 +71,21 @@ internal static class NodePackageCollector } } + AddAdditionalSourceRootPackages( + context, + projectInput.SourceRoots, + nodeModuleRoots, + lockData, + workspaceIndex, + workspaceDependencyIndices, + packages, + visited, + yarnPnpPresent, + cancellationToken); + foreach (var nodeModules in nodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal)) { - TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken); + TraverseDirectory(context, nodeModules, lockData, workspaceIndex, workspaceDependencyIndices, packages, visited, yarnPnpPresent, cancellationToken); } TraverseTarballs(context, projectInput.Tarballs, packages, visited, yarnPnpPresent, cancellationToken); @@ -76,6 +109,154 @@ internal static class NodePackageCollector return packages; } + private static void AddAdditionalSourceRootPackages( + LanguageAnalyzerContext context, + IReadOnlyList sourceRoots, + HashSet nodeModuleRoots, + NodeLockData lockData, + NodeWorkspaceIndex workspaceIndex, + IReadOnlyDictionary workspaceDependencyIndices, + List packages, + HashSet visited, + bool yarnPnpPresent, + CancellationToken cancellationToken) + { + if (sourceRoots.Count <= 1) + { + return; + } + + var rootPath = Path.GetFullPath(context.RootPath); + var comparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + + foreach (var sourceRoot in sourceRoots) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(sourceRoot)) + { + continue; + } + + var fullSourceRoot = Path.GetFullPath(sourceRoot); + if (comparer.Equals(fullSourceRoot, rootPath)) + { + continue; + } + + foreach (var packageDirectory in DiscoverPackageDirectories(fullSourceRoot, maxDepth: 4, maxDirsVisited: 8000, maxPackages: 64, cancellationToken)) + { + ProcessPackageDirectory( + context, + packageDirectory, + lockData, + workspaceIndex, + workspaceDependencyIndices, + includeNestedNodeModules: false, + packages, + visited, + yarnPnpPresent, + cancellationToken); + + var nestedNodeModules = Path.Combine(packageDirectory, "node_modules"); + if (Directory.Exists(nestedNodeModules)) + { + nodeModuleRoots.Add(nestedNodeModules); + } + } + } + } + + private static IReadOnlyList DiscoverPackageDirectories( + string rootPath, + int maxDepth, + int maxDirsVisited, + int maxPackages, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(rootPath) || !Directory.Exists(rootPath)) + { + return Array.Empty(); + } + + var comparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var visited = new HashSet(comparer); + var results = new List(); + var queue = new Queue<(string Path, int Depth)>(); + + queue.Enqueue((rootPath, 0)); + visited.Add(rootPath); + + var enumerationOptions = new EnumerationOptions + { + RecurseSubdirectories = false, + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.Device + }; + + var dirsVisited = 0; + while (queue.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + + var (current, depth) = queue.Dequeue(); + dirsVisited++; + if (dirsVisited > maxDirsVisited) + { + break; + } + + if (File.Exists(Path.Combine(current, "package.json"))) + { + results.Add(current); + if (results.Count >= maxPackages) + { + break; + } + } + + if (depth >= maxDepth) + { + continue; + } + + IEnumerable children; + try + { + children = Directory.EnumerateDirectories(current, "*", enumerationOptions); + } + catch + { + continue; + } + + foreach (var child in children.OrderBy(static p => p, StringComparer.Ordinal)) + { + var name = Path.GetFileName(child); + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + if (name.Equals("node_modules", StringComparison.OrdinalIgnoreCase) || + name.Equals(".pnpm", StringComparison.OrdinalIgnoreCase) || + name.Equals(".yarn", StringComparison.OrdinalIgnoreCase) || + name.Equals(".git", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (visited.Add(child)) + { + queue.Enqueue((child, depth + 1)); + } + } + } + + results.Sort(StringComparer.Ordinal); + return results; + } + /// /// Filters out packages that are declared-only (no on-disk evidence) when Yarn PnP data is available. /// Only emits packages that are actually resolved in the PnP resolution map. @@ -113,7 +294,13 @@ internal static class NodePackageCollector private static void AttachImports(LanguageAnalyzerContext context, List packages, CancellationToken cancellationToken) { - foreach (var package in packages) + const int maxFilesPerPackage = 500; + const long maxBytesPerPackage = 5L * 1024 * 1024; + const long maxFileBytes = 512L * 1024; + const int maxDepth = 20; + + foreach (var package in packages.Where(static p => string.IsNullOrEmpty(p.RelativePathNormalized) || p.IsWorkspaceMember) + .OrderBy(static p => p.RelativePathNormalized, StringComparer.Ordinal)) { cancellationToken.ThrowIfCancellationRequested(); @@ -126,10 +313,41 @@ internal static class NodePackageCollector continue; } - foreach (var file in EnumerateSourceFiles(packageRoot)) + var filesScanned = 0; + long bytesScanned = 0; + var capped = false; + + foreach (var file in EnumerateSourceFiles(packageRoot, maxDepth)) { cancellationToken.ThrowIfCancellationRequested(); + if (filesScanned >= maxFilesPerPackage || bytesScanned >= maxBytesPerPackage) + { + capped = true; + break; + } + + long length; + try + { + length = new FileInfo(file).Length; + } + catch + { + continue; + } + + if (length <= 0 || length > maxFileBytes) + { + continue; + } + + if (bytesScanned + length > maxBytesPerPackage) + { + capped = true; + break; + } + string content; try { @@ -140,6 +358,9 @@ internal static class NodePackageCollector continue; } + bytesScanned += length; + filesScanned++; + var relativeSource = context.GetRelativePath(file).Replace(Path.DirectorySeparatorChar, '/'); var imports = NodeImportWalker.AnalyzeImports(context.RootPath, relativeSource, content); foreach (var edge in imports) @@ -147,6 +368,11 @@ internal static class NodePackageCollector package.AddImport(edge); } } + + if (capped) + { + package.MarkImportScanSkipped(filesScanned, bytesScanned); + } } } @@ -237,27 +463,100 @@ internal static class NodePackageCollector return merged; } - private static IEnumerable EnumerateSourceFiles(string root) + private static IEnumerable EnumerateSourceFiles(string root, int maxDepth) { - foreach (var extension in new[] { ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx" }) + var extensions = new HashSet(StringComparer.OrdinalIgnoreCase) { - foreach (var file in Directory.EnumerateFiles(root, "*" + extension, new EnumerationOptions - { - RecurseSubdirectories = true, - MatchCasing = MatchCasing.CaseInsensitive, - IgnoreInaccessible = true - })) + ".js", + ".jsx", + ".mjs", + ".cjs", + ".ts", + ".tsx", + ".mts", + ".cts" + }; + + var pathComparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var stack = new Stack<(string Path, int Depth)>(); + stack.Push((root, 0)); + + while (stack.Count > 0) + { + var (current, depth) = stack.Pop(); + + IEnumerable files; + try { - yield return file; + files = Directory.EnumerateFiles(current, "*", SearchOption.TopDirectoryOnly); + } + catch + { + files = Array.Empty(); + } + + foreach (var file in files.OrderBy(static f => f, pathComparer)) + { + var ext = Path.GetExtension(file); + if (!string.IsNullOrWhiteSpace(ext) && extensions.Contains(ext)) + { + yield return file; + } + } + + if (depth >= maxDepth) + { + continue; + } + + IEnumerable dirs; + try + { + dirs = Directory.EnumerateDirectories(current, "*", SearchOption.TopDirectoryOnly); + } + catch + { + dirs = Array.Empty(); + } + + var ordered = dirs + .Where(static d => !ShouldSkipImportDirectory(Path.GetFileName(d))) + .OrderBy(static d => d, pathComparer) + .ToArray(); + + for (var i = ordered.Length - 1; i >= 0; i--) + { + stack.Push((ordered[i], depth + 1)); } } } + private static bool ShouldSkipImportDirectory(string? name) + { + if (string.IsNullOrWhiteSpace(name)) + { + return true; + } + + if (string.Equals(name, "node_modules", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return ShouldSkipDirectory(name); + } + private static void TraverseDirectory( LanguageAnalyzerContext context, string directory, NodeLockData lockData, NodeWorkspaceIndex workspaceIndex, + IReadOnlyDictionary workspaceDependencyIndices, List packages, HashSet visited, bool yarnPnpPresent, @@ -285,7 +584,7 @@ internal static class NodePackageCollector if (string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase)) { - TraversePnpmStore(context, child, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken); + TraversePnpmStore(context, child, lockData, workspaceIndex, workspaceDependencyIndices, packages, visited, yarnPnpPresent, cancellationToken); continue; } @@ -293,12 +592,12 @@ internal static class NodePackageCollector { foreach (var scoped in Directory.EnumerateDirectories(child)) { - ProcessPackageDirectory(context, scoped, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, yarnPnpPresent, cancellationToken); + ProcessPackageDirectory(context, scoped, lockData, workspaceIndex, workspaceDependencyIndices, includeNestedNodeModules: true, packages, visited, yarnPnpPresent, cancellationToken); } continue; } - ProcessPackageDirectory(context, child, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, yarnPnpPresent, cancellationToken); + ProcessPackageDirectory(context, child, lockData, workspaceIndex, workspaceDependencyIndices, includeNestedNodeModules: true, packages, visited, yarnPnpPresent, cancellationToken); } } @@ -307,6 +606,7 @@ internal static class NodePackageCollector string pnpmDirectory, NodeLockData lockData, NodeWorkspaceIndex workspaceIndex, + IReadOnlyDictionary workspaceDependencyIndices, List packages, HashSet visited, bool yarnPnpPresent, @@ -319,7 +619,7 @@ internal static class NodePackageCollector var nestedNodeModules = Path.Combine(storeEntry, "node_modules"); if (Directory.Exists(nestedNodeModules)) { - TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken); + TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, workspaceDependencyIndices, packages, visited, yarnPnpPresent, cancellationToken); } } } @@ -329,6 +629,7 @@ internal static class NodePackageCollector string directory, NodeLockData lockData, NodeWorkspaceIndex workspaceIndex, + IReadOnlyDictionary workspaceDependencyIndices, bool includeNestedNodeModules, List packages, HashSet visited, @@ -343,14 +644,14 @@ internal static class NodePackageCollector // Already processed this path. if (includeNestedNodeModules) { - TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken); + TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, workspaceDependencyIndices, packages, visited, yarnPnpPresent, cancellationToken); } return; } if (File.Exists(packageJsonPath)) { - var package = TryCreatePackage(context, packageJsonPath, relativeDirectory, lockData, workspaceIndex, yarnPnpPresent, cancellationToken); + var package = TryCreatePackage(context, packageJsonPath, relativeDirectory, lockData, workspaceIndex, workspaceDependencyIndices, yarnPnpPresent, cancellationToken); if (package is not null) { packages.Add(package); @@ -359,7 +660,7 @@ internal static class NodePackageCollector if (includeNestedNodeModules) { - TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken); + TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, workspaceDependencyIndices, packages, visited, yarnPnpPresent, cancellationToken); } } @@ -368,13 +669,14 @@ internal static class NodePackageCollector string directory, NodeLockData lockData, NodeWorkspaceIndex workspaceIndex, + IReadOnlyDictionary workspaceDependencyIndices, List packages, HashSet visited, bool yarnPnpPresent, CancellationToken cancellationToken) { var nestedNodeModules = Path.Combine(directory, "node_modules"); - TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, yarnPnpPresent, cancellationToken); + TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, workspaceDependencyIndices, packages, visited, yarnPnpPresent, cancellationToken); } private static void TraverseTarballs( @@ -602,27 +904,57 @@ internal static class NodePackageCollector string relativeDirectory, NodeLockData lockData, NodeWorkspaceIndex workspaceIndex, + IReadOnlyDictionary workspaceDependencyIndices, bool yarnPnpPresent, CancellationToken cancellationToken) { try { - using var stream = File.OpenRead(packageJsonPath); - using var document = JsonDocument.Parse(stream); + const int maxHashBytes = 1_048_576; - var root = document.RootElement; - return TryCreatePackageFromJson( - context, - root, - relativeDirectory, - BuildLocator(relativeDirectory), - context.UsageHints.IsPathUsed(packageJsonPath), - cancellationToken, - lockData, - workspaceIndex, - packageJsonPath, - packageSha256: null, - yarnPnpPresent: yarnPnpPresent); + var sha256Hex = default(string); + JsonDocument document; + + var info = new FileInfo(packageJsonPath); + if (info is { Exists: true } && info.Length is > 0 and <= maxHashBytes) + { + var bytes = File.ReadAllBytes(packageJsonPath); + if (bytes.Length is > 0 and <= maxHashBytes) + { + sha256Hex = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + } + + var parseBytes = bytes; + if (bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) + { + parseBytes = bytes[3..]; + } + + document = JsonDocument.Parse(parseBytes); + } + else + { + using var stream = File.OpenRead(packageJsonPath); + document = JsonDocument.Parse(stream); + } + + using (document) + { + var root = document.RootElement; + return TryCreatePackageFromJson( + context, + root, + relativeDirectory, + BuildLocator(relativeDirectory), + context.UsageHints.IsPathUsed(packageJsonPath), + cancellationToken, + lockData, + workspaceIndex, + workspaceDependencyIndices, + packageJsonPath, + packageSha256: sha256Hex, + yarnPnpPresent: yarnPnpPresent); + } } catch (IOException) { @@ -643,6 +975,7 @@ internal static class NodePackageCollector CancellationToken cancellationToken, NodeLockData? lockData = null, NodeWorkspaceIndex? workspaceIndex = null, + IReadOnlyDictionary? workspaceDependencyIndices = null, string? packageJsonPath = null, string? packageSha256 = null, bool yarnPnpPresent = false) @@ -675,21 +1008,31 @@ internal static class NodePackageCollector isPrivate = privateElement.GetBoolean(); } - var lockEntry = lockData?.TryGet(relativeDirectory, name, out var entry) == true ? entry : null; + var lockEntry = lockData?.TryGet(relativeDirectory, name, version, out var entry) == true ? entry : null; var lockLocator = BuildLockLocator(lockEntry); var lockSource = lockEntry?.Source; - // Get scope from lock entry (populated by NodeLockData from package.json) - // or from the dependency index directly if this is a root package - NodeDependencyScope? scope = lockEntry?.Scope; - var isOptional = lockEntry?.IsOptional ?? false; - if (scope is null && lockData?.DependencyIndex is { } dependencyIndex) + NodeDependencyScope? scope = null; + var isOptional = false; + + if (workspaceIndex is not null + && workspaceDependencyIndices is not null + && workspaceIndex.TryGetOwningWorkspace(relativeDirectory, out var owningWorkspace) + && workspaceDependencyIndices.TryGetValue(owningWorkspace, out var owningIndex) + && owningIndex.TryGetScope(name, out var workspaceScope)) { - if (dependencyIndex.TryGetScope(name, out var foundScope)) - { - scope = foundScope; - isOptional = foundScope == NodeDependencyScope.Optional; - } + scope = workspaceScope; + isOptional = workspaceScope == NodeDependencyScope.Optional; + } + else if (lockData?.DependencyIndex is { } dependencyIndex && dependencyIndex.TryGetScope(name, out var rootScope)) + { + scope = rootScope; + isOptional = rootScope == NodeDependencyScope.Optional; + } + else if (lockEntry?.Scope is { } lockScope) + { + scope = lockScope; + isOptional = lockEntry?.IsOptional ?? false; } // Extract license from package.json diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs index 262df68e9..098a7ab9a 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs @@ -4,6 +4,10 @@ namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; internal sealed class NodeWorkspaceIndex { + private const int WorkspaceGlobMaxDepth = 10; + private const int WorkspaceGlobMaxDirsVisited = 20_000; + private const int WorkspaceGlobMaxMembers = 2_000; + private readonly string _rootPath; private readonly HashSet _workspacePaths; private readonly Dictionary _workspaceByName; @@ -18,7 +22,8 @@ internal sealed class NodeWorkspaceIndex public static NodeWorkspaceIndex Create(string rootPath) { var normalizedRoot = Path.GetFullPath(rootPath); - var workspacePaths = new HashSet(StringComparer.Ordinal); + var pathComparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var workspacePaths = new HashSet(pathComparer); var workspaceByName = new Dictionary(StringComparer.OrdinalIgnoreCase); var packageJsonPath = Path.Combine(normalizedRoot, "package.json"); @@ -112,6 +117,44 @@ internal sealed class NodeWorkspaceIndex return false; } + public bool TryGetOwningWorkspace(string relativePath, out string workspaceRoot) + { + workspaceRoot = string.Empty; + if (string.IsNullOrWhiteSpace(relativePath)) + { + return false; + } + + var normalized = NormalizeRelative(relativePath); + if (string.IsNullOrWhiteSpace(normalized)) + { + return false; + } + + if (_workspacePaths.Contains(normalized)) + { + workspaceRoot = normalized; + return true; + } + + for (var i = normalized.Length - 1; i > 0; i--) + { + if (normalized[i] != '/') + { + continue; + } + + var candidate = normalized[..i]; + if (_workspacePaths.Contains(candidate)) + { + workspaceRoot = candidate; + return true; + } + } + + return false; + } + public bool TryGetWorkspacePathByName(string packageName, out string? relativePath) => _workspaceByName.TryGetValue(packageName, out relativePath); @@ -220,31 +263,133 @@ internal sealed class NodeWorkspaceIndex private static IEnumerable ExpandPattern(string rootPath, string pattern) { var cleanedPattern = pattern.Replace('\\', '/').Trim(); - if (cleanedPattern.EndsWith("/*", StringComparison.Ordinal)) + if (string.IsNullOrWhiteSpace(cleanedPattern)) { - var baseSegment = cleanedPattern[..^2]; - var baseAbsolute = CombineAndNormalize(rootPath, baseSegment); - if (baseAbsolute is null || !Directory.Exists(baseAbsolute)) - { - yield break; - } - - foreach (var directory in Directory.EnumerateDirectories(baseAbsolute)) - { - var normalized = NormalizeRelative(Path.GetRelativePath(rootPath, directory)); - yield return normalized; - } + yield break; } - else + + if (cleanedPattern.StartsWith('!')) { - var absolute = CombineAndNormalize(rootPath, cleanedPattern); - if (absolute is null || !Directory.Exists(absolute)) + // Exclusion patterns are ignored (workspace discovery is conservative and bounded). + yield break; + } + + cleanedPattern = cleanedPattern.TrimStart('.', '/'); + if (string.IsNullOrWhiteSpace(cleanedPattern)) + { + yield break; + } + + var segments = cleanedPattern.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length == 0) + { + yield break; + } + + var pathComparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + var results = new HashSet(pathComparer); + var visitedStates = new HashSet(pathComparer); + + var queue = new Queue<(string AbsolutePath, int SegmentIndex, int Depth)>(); + queue.Enqueue((rootPath, 0, 0)); + + var dirsVisited = 0; + while (queue.Count > 0) + { + if (results.Count >= WorkspaceGlobMaxMembers || dirsVisited >= WorkspaceGlobMaxDirsVisited) { - yield break; + break; } - var normalized = NormalizeRelative(Path.GetRelativePath(rootPath, absolute)); - yield return normalized; + var (absolutePath, segmentIndex, depth) = queue.Dequeue(); + if (!visitedStates.Add($"{segmentIndex}:{absolutePath}")) + { + continue; + } + + if (segmentIndex >= segments.Length) + { + var relative = NormalizeRelative(Path.GetRelativePath(rootPath, absolutePath)); + if (string.IsNullOrWhiteSpace(relative)) + { + continue; + } + + var packageJsonPath = Path.Combine(absolutePath, "package.json"); + if (!File.Exists(packageJsonPath)) + { + continue; + } + + results.Add(relative); + continue; + } + + var segment = segments[segmentIndex]; + if (segment == "**") + { + queue.Enqueue((absolutePath, segmentIndex + 1, depth)); + + if (depth >= WorkspaceGlobMaxDepth) + { + continue; + } + + foreach (var child in EnumerateDirectoriesSorted(absolutePath)) + { + if (dirsVisited++ >= WorkspaceGlobMaxDirsVisited) + { + break; + } + + queue.Enqueue((child, segmentIndex, depth + 1)); + } + + continue; + } + + if (segment == "*") + { + if (depth >= WorkspaceGlobMaxDepth) + { + continue; + } + + foreach (var child in EnumerateDirectoriesSorted(absolutePath)) + { + if (dirsVisited++ >= WorkspaceGlobMaxDirsVisited) + { + break; + } + + queue.Enqueue((child, segmentIndex + 1, depth + 1)); + } + + continue; + } + + if (IsExcludedSegment(segment)) + { + continue; + } + + var nextAbsolute = CombineAndNormalize(absolutePath, segment); + if (nextAbsolute is null || !Directory.Exists(nextAbsolute)) + { + continue; + } + + if (!IsUnderRoot(rootPath, nextAbsolute)) + { + continue; + } + + queue.Enqueue((nextAbsolute, segmentIndex + 1, depth + 1)); + } + + foreach (var member in results.OrderBy(static path => path, pathComparer)) + { + yield return NormalizeRelative(member); } } @@ -254,6 +399,35 @@ internal sealed class NodeWorkspaceIndex return IsUnderRoot(rootPath, candidate) ? candidate : null; } + private static IReadOnlyList EnumerateDirectoriesSorted(string absolutePath) + { + try + { + return Directory.EnumerateDirectories(absolutePath) + .Where(static path => !IsExcludedDirectoryName(Path.GetFileName(path))) + .OrderBy(static path => path, OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal) + .ToArray(); + } + catch (UnauthorizedAccessException) + { + return Array.Empty(); + } + catch (DirectoryNotFoundException) + { + return Array.Empty(); + } + catch (IOException) + { + return Array.Empty(); + } + } + + private static bool IsExcludedSegment(string segment) + => string.Equals(segment, "node_modules", StringComparison.OrdinalIgnoreCase); + + private static bool IsExcludedDirectoryName(string? name) + => string.Equals(name, "node_modules", StringComparison.OrdinalIgnoreCase); + private static string NormalizeRelative(string relativePath) { if (string.IsNullOrEmpty(relativePath) || relativePath == ".") diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs index d301e11bf..694e39440 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs @@ -36,6 +36,8 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer usedByEntrypoint: package.IsUsedByEntrypoint); } + EmitDeclaredOnlyPackages(writer, lockData, packages); + var observation = NodePhase22Analyzer.Analyze(context, cancellationToken); if (observation.HasRecords) { @@ -84,4 +86,274 @@ public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer writer.AddRange(envWarnings); } } + + private static void EmitDeclaredOnlyPackages( + LanguageComponentWriter writer, + NodeLockData lockData, + IReadOnlyList packages) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentNullException.ThrowIfNull(lockData); + ArgumentNullException.ThrowIfNull(packages); + + if (lockData.DeclaredPackages.Count == 0) + { + return; + } + + var installedByNameVersion = new HashSet(StringComparer.OrdinalIgnoreCase); + var installedNames = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var package in packages) + { + installedNames.Add(package.Name); + installedByNameVersion.Add($"{package.Name}@{package.Version}"); + } + + foreach (var entry in lockData.DeclaredPackages.OrderBy(static e => e.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(static e => e.Version ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static e => e.Source, StringComparer.OrdinalIgnoreCase) + .ThenBy(static e => e.Locator ?? string.Empty, StringComparer.OrdinalIgnoreCase)) + { + if (string.IsNullOrWhiteSpace(entry.Name)) + { + continue; + } + + var versionSpec = ResolveVersionSpec(lockData, entry); + + var hasResolvedVersion = entry.Source is not null + && !string.Equals(entry.Source, "package.json", StringComparison.OrdinalIgnoreCase) + && IsConcreteNpmVersion(entry.Version); + + var resolvedVersion = hasResolvedVersion ? entry.Version!.Trim() : null; + + var backedByInventory = resolvedVersion is not null + ? installedByNameVersion.Contains($"{entry.Name}@{resolvedVersion}") + : installedNames.Contains(entry.Name); + + if (backedByInventory) + { + continue; + } + + var declaredLocator = BuildDeclaredLocator(entry); + var scopeValue = entry.Scope is { } scope ? scope.ToString().ToLowerInvariant() : null; + var metadata = new List>(8) + { + new("declaredOnly", "true"), + new("declared.source", entry.Source), + new("declared.locator", declaredLocator), + new("declared.versionSpec", versionSpec), + new("declared.sourceType", ClassifyDeclaredSourceType(versionSpec)), + }; + + if (!string.IsNullOrWhiteSpace(scopeValue)) + { + metadata.Add(new KeyValuePair("declared.scope", scopeValue)); + } + + if (resolvedVersion is not null) + { + metadata.Add(new KeyValuePair("declared.resolvedVersion", resolvedVersion)); + } + + if (entry.IntegrityMissing) + { + metadata.Add(new KeyValuePair("lockIntegrityMissing", "true")); + if (!string.IsNullOrWhiteSpace(entry.IntegrityMissingReason)) + { + metadata.Add(new KeyValuePair("lockIntegrityMissingReason", entry.IntegrityMissingReason)); + } + } + + var evidence = new[] + { + new LanguageComponentEvidence( + Kind: LanguageEvidenceKind.Metadata, + Source: "node.declared", + Locator: declaredLocator, + Value: null, + Sha256: null) + }; + + if (resolvedVersion is not null) + { + var purl = BuildNpmPurl(entry.Name, resolvedVersion); + writer.AddFromPurl( + analyzerId: "node", + purl: purl, + name: entry.Name, + version: resolvedVersion, + type: "npm", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: false); + continue; + } + + var componentKey = LanguageExplicitKey.Create("node", "npm", entry.Name, versionSpec, declaredLocator); + writer.AddFromExplicitKey( + analyzerId: "node", + componentKey: componentKey, + purl: null, + name: entry.Name, + version: null, + type: "npm", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: false); + } + } + + private static string BuildDeclaredLocator(NodeLockEntry entry) + { + if (string.IsNullOrWhiteSpace(entry.Source)) + { + return string.IsNullOrWhiteSpace(entry.Locator) ? "package.json" : entry.Locator!.Trim(); + } + + if (string.IsNullOrWhiteSpace(entry.Locator)) + { + return entry.Source; + } + + var locator = entry.Locator!.Trim(); + if (entry.Source.Equals("package.json", StringComparison.OrdinalIgnoreCase) && + locator.StartsWith("package.json#", StringComparison.OrdinalIgnoreCase)) + { + return locator; + } + + return $"{entry.Source}:{locator}"; + } + + private static string ClassifyDeclaredSourceType(string spec) + { + if (string.IsNullOrWhiteSpace(spec)) + { + return "unknown"; + } + + var value = spec.Trim(); + + if (value.StartsWith("workspace:", StringComparison.OrdinalIgnoreCase)) + { + return "workspace"; + } + + if (value.StartsWith("link:", StringComparison.OrdinalIgnoreCase)) + { + return "link"; + } + + if (value.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + { + return "file"; + } + + if (value.StartsWith("git+", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("git://", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("github:", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("gitlab:", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("bitbucket:", StringComparison.OrdinalIgnoreCase)) + { + return "git"; + } + + if (value.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + value.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + return "tarball"; + } + + if (value is "latest" or "next" or "beta" or "alpha" or "canary") + { + return "tag"; + } + + if (value.Length > 0 && value[0] == '.' || + value.Length > 0 && value[0] == '/' || + value.Contains('\\')) + { + return "path"; + } + + return "range"; + } + + private static string ResolveVersionSpec(NodeLockData lockData, NodeLockEntry entry) + { + if (lockData.DependencyIndex.TryGetDeclaration(entry.Name, out var declaration) && !string.IsNullOrWhiteSpace(declaration?.VersionRange)) + { + return declaration!.VersionRange!; + } + + if (!string.IsNullOrWhiteSpace(entry.Version)) + { + return entry.Version!.Trim(); + } + + return "*"; + } + + private static bool IsConcreteNpmVersion(string? version) + { + if (string.IsNullOrWhiteSpace(version)) + { + return false; + } + + version = version.Trim(); + if (version.StartsWith('^') || version.StartsWith('~') || version.StartsWith('>') || version.StartsWith('<')) + { + return false; + } + + if (version.Contains(' ') || version.Contains('*') || version.Contains('|') || version.Contains(':') || version.Contains('/')) + { + return false; + } + + var hasDigit = false; + foreach (var ch in version) + { + if (char.IsDigit(ch)) + { + hasDigit = true; + continue; + } + + if (ch is '.' or '-' or '+' or '_' || char.IsLetter(ch)) + { + continue; + } + + return false; + } + + return hasDigit; + } + + private static string BuildNpmPurl(string name, string version) + { + var normalizedName = NormalizeNpmName(name); + return $"pkg:npm/{normalizedName}@{version}"; + } + + private static string NormalizeNpmName(string name) + { + if (string.IsNullOrWhiteSpace(name)) + { + return name; + } + + name = name.Trim(); + if (name[0] == '@') + { + var scopeAndName = name[1..]; + return $"%40{scopeAndName}"; + } + + return name; + } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md index 93c226c9f..0aaefba57 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md @@ -14,3 +14,19 @@ | SCANNER-ANALYZERS-NODE-22-010 | DONE | Runtime evidence hooks (ESM loader/CJS require) with path scrubbing and hashed loader IDs; ingestion to runtime-* records. | 2025-12-01 | | SCANNER-ANALYZERS-NODE-22-011 | DONE | Packaged plug-in manifest (0.1.0) with runtime hooks; CLI/offline docs refreshed. | 2025-12-01 | | SCANNER-ANALYZERS-NODE-22-012 | DONE | Container filesystem adapter (layer roots) + NODE_OPTIONS/env warnings emitted. | 2025-12-01 | + +## Node Detection Gaps (Sprint 0406) + +| Task ID | Status | Notes | Updated (UTC) | +| --- | --- | --- | --- | +| SCAN-NODE-406-001 | DONE | Emit declared-only components (explicit-key via LanguageExplicitKey; no range-as-version PURLs; sourceType metadata). | 2025-12-13 | +| SCAN-NODE-406-002 | DONE | Multi-version lock correctness + `(name,version)` matching. | 2025-12-13 | +| SCAN-NODE-406-003 | DONE | Yarn Berry (v2/v3) lock parsing. | 2025-12-13 | +| SCAN-NODE-406-004 | DONE | Harden pnpm lock parsing (integrity-missing, snapshots). | 2025-12-13 | +| SCAN-NODE-406-005 | DONE | Fix package-lock nested node_modules naming. | 2025-12-13 | +| SCAN-NODE-406-006 | DONE | Workspace glob expansion (`*`/`**`) + bounds. | 2025-12-13 | +| SCAN-NODE-406-007 | DONE | Workspace-aware dependency scopes. | 2025-12-13 | +| SCAN-NODE-406-008 | DONE | Import scanning correctness + bounds. | 2025-12-13 | +| SCAN-NODE-406-009 | DONE | Deterministic package.json hashing for on-disk packages + fixtures. | 2025-12-13 | +| SCAN-NODE-406-010 | DONE | Fixtures + goldens: lock-only package-lock/yarn-berry/pnpm, workspace glob (`*`/`**`), container app-root discovery. | 2025-12-13 | +| SCAN-NODE-406-011 | DONE | Docs + offline benchmark (Node contract doc + new bench scenario + import-scan metrics). | 2025-12-13 | diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Entrypoints/PythonEntrypointDiscovery.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Entrypoints/PythonEntrypointDiscovery.cs index d7d58b6b7..c2616fbb7 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Entrypoints/PythonEntrypointDiscovery.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Entrypoints/PythonEntrypointDiscovery.cs @@ -87,25 +87,16 @@ internal sealed partial class PythonEntrypointDiscovery { cancellationToken.ThrowIfCancellationRequested(); - var absolutePath = file.AbsolutePath; - if (file.IsFromArchive) + try { - continue; // Can't read from archive directly yet - } - - var fullPath = Path.Combine(_rootPath, absolutePath); - if (!File.Exists(fullPath)) - { - fullPath = absolutePath; - if (!File.Exists(fullPath)) + using var stream = await _vfs.OpenReadAsync(file.VirtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) { continue; } - } - try - { - var content = await File.ReadAllTextAsync(fullPath, cancellationToken).ConfigureAwait(false); + using var reader = new StreamReader(stream); + var content = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); ParseEntryPointsTxt(content, file.VirtualPath); } catch (IOException) @@ -225,7 +216,7 @@ internal sealed partial class PythonEntrypointDiscovery { cancellationToken.ThrowIfCancellationRequested(); - if (file.VirtualPath == "__main__.py") + if (string.Equals(file.AbsolutePath, "__main__.py", StringComparison.OrdinalIgnoreCase)) { _entrypoints.Add(new PythonEntrypoint( Name: "__main__", diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/Adapters/PipEditableAdapter.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/Adapters/PipEditableAdapter.cs index 5e1670d92..56516d1b5 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/Adapters/PipEditableAdapter.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/Adapters/PipEditableAdapter.cs @@ -48,11 +48,15 @@ internal sealed partial class PipEditableAdapter : IPythonPackagingAdapter continue; } - // Look for .egg-info in the target - var (version, metadata, topLevel) = await ReadEggInfoAsync(vfs, targetPath, packageName, cancellationToken).ConfigureAwait(false); + // The editable target path in .egg-link can be absolute and host-specific. + // Prefer the VFS-mounted editable tree under the packageName prefix. + var editableRoot = packageName; + + // Look for .egg-info in the editable root + var (version, metadata, topLevel) = await ReadEggInfoAsync(vfs, editableRoot, packageName, cancellationToken).ConfigureAwait(false); // Also look for pyproject.toml for additional metadata - var pyprojectInfo = await ReadPyprojectAsync(vfs, targetPath, cancellationToken).ConfigureAwait(false); + var pyprojectInfo = await ReadPyprojectAsync(vfs, editableRoot, cancellationToken).ConfigureAwait(false); if (pyprojectInfo.Name is not null) { @@ -79,7 +83,7 @@ internal sealed partial class PipEditableAdapter : IPythonPackagingAdapter Extras: ImmutableArray.Empty, RecordFiles: ImmutableArray.Empty, InstallerTool: "pip", - EditableTarget: targetPath, + EditableTarget: editableRoot, IsDirectDependency: true, // Editable installs are always direct Confidence: PythonPackageConfidence.High); } @@ -110,7 +114,7 @@ internal sealed partial class PipEditableAdapter : IPythonPackagingAdapter private static async Task<(string? Version, Dictionary Metadata, ImmutableArray TopLevel)> ReadEggInfoAsync( PythonVirtualFileSystem vfs, - string targetPath, + string editableRoot, string packageName, CancellationToken cancellationToken) { @@ -119,8 +123,7 @@ internal sealed partial class PipEditableAdapter : IPythonPackagingAdapter var topLevel = ImmutableArray.Empty; // Look for .egg-info directory - var eggInfoPattern = $"{packageName}.egg-info"; - var eggInfoFiles = vfs.EnumerateFiles(targetPath, "*.egg-info/PKG-INFO").ToList(); + var eggInfoFiles = vfs.EnumerateFiles(editableRoot, "*.egg-info/PKG-INFO").ToList(); PythonVirtualFile? pkgInfoFile = null; foreach (var file in eggInfoFiles) @@ -204,10 +207,10 @@ internal sealed partial class PipEditableAdapter : IPythonPackagingAdapter private static async Task<(string? Name, string? Version)> ReadPyprojectAsync( PythonVirtualFileSystem vfs, - string targetPath, + string editableRoot, CancellationToken cancellationToken) { - var pyprojectPath = $"{targetPath}/pyproject.toml"; + var pyprojectPath = $"{editableRoot}/pyproject.toml"; try { diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/PythonPackageDiscovery.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/PythonPackageDiscovery.cs index 5a72dd3c8..bb2136a8f 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/PythonPackageDiscovery.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/PythonPackageDiscovery.cs @@ -37,12 +37,16 @@ internal sealed class PythonPackageDiscovery var errors = new List(); var searchPaths = new List(); - // Gather all search paths from VFS - searchPaths.AddRange(vfs.SitePackagesPaths); - searchPaths.AddRange(vfs.SourceTreeRoots); - searchPaths.AddRange(vfs.EditablePaths); + // Gather all search paths from VFS (ordered by intended precedence). + // Later paths overwrite earlier ones on equal confidence. + searchPaths.Add(string.Empty); // workspace root (pyproject/locks/etc.) + searchPaths.AddRange(vfs.SourceTreeRoots.OrderBy(static path => path, StringComparer.Ordinal)); + searchPaths.AddRange(vfs.EditablePaths.OrderBy(static path => path, StringComparer.Ordinal)); + searchPaths.AddRange(vfs.SitePackagesPaths.OrderBy(static path => path, StringComparer.Ordinal)); + searchPaths.AddRange(vfs.ZipArchivePaths.OrderBy(static path => path, StringComparer.Ordinal)); - foreach (var path in searchPaths.Distinct()) + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var path in searchPaths.Where(p => seen.Add(p))) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionVfsLoader.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionVfsLoader.cs new file mode 100644 index 000000000..113d5b045 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/PythonDistributionVfsLoader.cs @@ -0,0 +1,937 @@ +using System.Buffers; +using System.Globalization; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem; +using Packaging = StellaOps.Scanner.Analyzers.Lang.Python.Internal.Packaging; + +namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal; + +internal static class PythonDistributionVfsLoader +{ + public static async Task LoadAsync( + LanguageAnalyzerContext context, + PythonVirtualFileSystem vfs, + Packaging.PythonPackageInfo package, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(vfs); + + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(package.MetadataPath)) + { + return null; + } + + var isEggInfo = package.Kind == Packaging.PythonPackageKind.Egg; + var metadataName = isEggInfo ? "PKG-INFO" : "METADATA"; + var recordName = isEggInfo ? "installed-files.txt" : "RECORD"; + + var metadataVirtualPath = $"{package.MetadataPath}/{metadataName}"; + if (!vfs.FileExists(metadataVirtualPath)) + { + return null; + } + + var metadataDocument = await PythonMetadataDocumentVfs.LoadAsync(vfs, metadataVirtualPath, cancellationToken).ConfigureAwait(false); + var name = (metadataDocument.GetFirst("Name") ?? package.Name)?.Trim(); + var version = (metadataDocument.GetFirst("Version") ?? package.Version)?.Trim(); + + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(version)) + { + return null; + } + + var normalizedName = PythonPathHelper.NormalizePackageName(name); + var purl = $"pkg:pypi/{normalizedName}@{version}"; + + var metadataEntries = new List>(); + var evidenceEntries = new List(); + + AppendMetadata(metadataEntries, "distInfoPath", NormalizeVfsPath(package.MetadataPath)); + AppendMetadata(metadataEntries, "name", name); + AppendMetadata(metadataEntries, "version", version); + AppendMetadata(metadataEntries, "normalizedName", normalizedName); + AppendMetadata(metadataEntries, "summary", metadataDocument.GetFirst("Summary")); + AppendMetadata(metadataEntries, "license", metadataDocument.GetFirst("License")); + AppendMetadata(metadataEntries, "licenseExpression", metadataDocument.GetFirst("License-Expression")); + AppendMetadata(metadataEntries, "homePage", metadataDocument.GetFirst("Home-page")); + AppendMetadata(metadataEntries, "author", metadataDocument.GetFirst("Author")); + AppendMetadata(metadataEntries, "authorEmail", metadataDocument.GetFirst("Author-email")); + AppendMetadata(metadataEntries, "projectUrl", metadataDocument.GetFirst("Project-URL")); + AppendMetadata(metadataEntries, "requiresPython", metadataDocument.GetFirst("Requires-Python")); + + AppendClassifiers(metadataEntries, metadataDocument); + + var requiresDist = metadataDocument.GetAll("Requires-Dist"); + if (requiresDist.Count > 0) + { + AppendMetadata(metadataEntries, "requiresDist", string.Join(';', requiresDist)); + } + + await AppendEntryPointsAsync(vfs, metadataEntries, $"{package.MetadataPath}/entry_points.txt", cancellationToken) + .ConfigureAwait(false); + + if (!isEggInfo) + { + await AppendWheelMetadataAsync(vfs, metadataEntries, $"{package.MetadataPath}/WHEEL", cancellationToken) + .ConfigureAwait(false); + } + + var installer = await ReadSingleLineAsync(vfs, $"{package.MetadataPath}/INSTALLER", cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(installer)) + { + AppendMetadata(metadataEntries, "installer", installer); + } + + await AppendDirectUrlAsync(context, vfs, metadataEntries, evidenceEntries, $"{package.MetadataPath}/direct_url.json", cancellationToken) + .ConfigureAwait(false); + + AddOptionalFileEvidence(context, vfs, evidenceEntries, metadataVirtualPath, metadataName); + AddOptionalFileEvidence(context, vfs, evidenceEntries, $"{package.MetadataPath}/WHEEL", "WHEEL"); + AddOptionalFileEvidence(context, vfs, evidenceEntries, $"{package.MetadataPath}/entry_points.txt", "entry_points.txt"); + AddOptionalFileEvidence(context, vfs, evidenceEntries, $"{package.MetadataPath}/INSTALLER", "INSTALLER"); + AddOptionalFileEvidence(context, vfs, evidenceEntries, $"{package.MetadataPath}/{recordName}", recordName); + AddOptionalFileEvidence(context, vfs, evidenceEntries, $"{package.MetadataPath}/direct_url.json", "direct_url.json"); + + var recordVirtualPath = $"{package.MetadataPath}/{recordName}"; + var recordEntries = await ReadRecordAsync(vfs, recordVirtualPath, cancellationToken).ConfigureAwait(false); + var recordVerification = await VerifyRecordAsync(vfs, package.MetadataPath, recordEntries, cancellationToken).ConfigureAwait(false); + + metadataEntries.Add(new KeyValuePair("record.totalEntries", recordVerification.TotalEntries.ToString(CultureInfo.InvariantCulture))); + metadataEntries.Add(new KeyValuePair("record.hashedEntries", recordVerification.HashedEntries.ToString(CultureInfo.InvariantCulture))); + metadataEntries.Add(new KeyValuePair("record.missingFiles", recordVerification.MissingFiles.ToString(CultureInfo.InvariantCulture))); + metadataEntries.Add(new KeyValuePair("record.hashMismatches", recordVerification.HashMismatches.ToString(CultureInfo.InvariantCulture))); + metadataEntries.Add(new KeyValuePair("record.ioErrors", recordVerification.IoErrors.ToString(CultureInfo.InvariantCulture))); + + if (recordVerification.UnsupportedAlgorithms.Count > 0) + { + AppendMetadata( + metadataEntries, + "record.unsupportedAlgorithms", + string.Join(';', recordVerification.UnsupportedAlgorithms.OrderBy(static a => a, StringComparer.OrdinalIgnoreCase))); + } + + evidenceEntries.AddRange(recordVerification.Evidence); + + AppendMetadata(metadataEntries, "provenance", isEggInfo ? "egg-info" : "dist-info"); + + var usedByEntrypoint = false; + + return new PythonDistribution( + name, + version, + purl, + metadataEntries, + evidenceEntries, + usedByEntrypoint); + } + + private static void AddOptionalFileEvidence( + LanguageAnalyzerContext context, + PythonVirtualFileSystem vfs, + ICollection evidence, + string virtualPath, + string source) + { + var file = vfs.GetFile(virtualPath); + if (file is null) + { + return; + } + + if (file.IsFromArchive && file.ArchivePath is not null) + { + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.File, + source, + PythonPathHelper.NormalizeRelative(context, file.ArchivePath), + Value: file.AbsolutePath, + Sha256: null)); + return; + } + + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.File, + source, + PythonPathHelper.NormalizeRelative(context, file.AbsolutePath), + Value: null, + Sha256: null)); + } + + private static void AppendClassifiers( + ICollection> metadata, + PythonMetadataDocumentVfs metadataDocument) + { + var classifiers = metadataDocument.GetAll("Classifier"); + if (classifiers.Count == 0) + { + return; + } + + var orderedClassifiers = classifiers + .Select(static classifier => classifier.Trim()) + .Where(static classifier => classifier.Length > 0) + .OrderBy(static classifier => classifier, StringComparer.Ordinal) + .ToArray(); + + if (orderedClassifiers.Length == 0) + { + return; + } + + AppendMetadata(metadata, "classifiers", string.Join(';', orderedClassifiers)); + + var licenseClassifierIndex = 0; + for (var index = 0; index < orderedClassifiers.Length; index++) + { + var classifier = orderedClassifiers[index]; + AppendMetadata(metadata, $"classifier[{index}]", classifier); + + if (classifier.StartsWith("License ::", StringComparison.OrdinalIgnoreCase)) + { + AppendMetadata(metadata, $"license.classifier[{licenseClassifierIndex}]", classifier); + licenseClassifierIndex++; + } + } + } + + private static async Task AppendEntryPointsAsync( + PythonVirtualFileSystem vfs, + ICollection> metadata, + string entryPointsVirtualPath, + CancellationToken cancellationToken) + { + if (!vfs.FileExists(entryPointsVirtualPath)) + { + return; + } + + string? content; + try + { + await using var stream = await vfs.OpenReadAsync(entryPointsVirtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return; + } + + using var reader = new StreamReader(stream, PythonEncoding.Utf8, detectEncodingFromByteOrderMarks: true); + content = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + return; + } + + var groups = new Dictionary>(StringComparer.OrdinalIgnoreCase); + string? currentGroup = null; + + foreach (var rawLine in content.Split('\n')) + { + var line = rawLine.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + { + continue; + } + + if (line.StartsWith('[') && line.EndsWith(']')) + { + currentGroup = line[1..^1].Trim(); + if (currentGroup.Length == 0) + { + currentGroup = null; + } + + continue; + } + + if (currentGroup is null) + { + continue; + } + + var separator = line.IndexOf('='); + if (separator <= 0) + { + continue; + } + + var name = line[..separator].Trim(); + var target = line[(separator + 1)..].Trim(); + if (name.Length == 0 || target.Length == 0) + { + continue; + } + + if (!groups.TryGetValue(currentGroup, out var list)) + { + list = new List<(string Name, string Target)>(); + groups[currentGroup] = list; + } + + list.Add((name, target)); + } + + foreach (var group in groups.OrderBy(static g => g.Key, StringComparer.OrdinalIgnoreCase)) + { + AppendMetadata(metadata, $"entryPoints.{group.Key}", string.Join(';', group.Value.Select(static ep => $"{ep.Name}={ep.Target}"))); + } + } + + private static async Task AppendWheelMetadataAsync( + PythonVirtualFileSystem vfs, + ICollection> metadata, + string wheelVirtualPath, + CancellationToken cancellationToken) + { + if (!vfs.FileExists(wheelVirtualPath)) + { + return; + } + + var values = new Dictionary(StringComparer.OrdinalIgnoreCase); + try + { + await using var stream = await vfs.OpenReadAsync(wheelVirtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return; + } + + using var reader = new StreamReader(stream, PythonEncoding.Utf8, detectEncodingFromByteOrderMarks: true); + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var separator = line.IndexOf(':'); + if (separator <= 0) + { + continue; + } + + var key = line[..separator].Trim(); + var value = line[(separator + 1)..].Trim(); + if (key.Length == 0 || value.Length == 0) + { + continue; + } + + values[key] = value; + } + } + catch (IOException) + { + return; + } + + if (values.TryGetValue("Wheel-Version", out var wheelVersion)) + { + AppendMetadata(metadata, "wheel.version", wheelVersion); + } + + if (values.TryGetValue("Tag", out var tags)) + { + AppendMetadata(metadata, "wheel.tags", tags); + } + + if (values.TryGetValue("Root-Is-Purelib", out var purelib)) + { + AppendMetadata(metadata, "wheel.rootIsPurelib", purelib); + } + + if (values.TryGetValue("Generator", out var generator)) + { + AppendMetadata(metadata, "wheel.generator", generator); + } + } + + private static async Task ReadSingleLineAsync( + PythonVirtualFileSystem vfs, + string virtualPath, + CancellationToken cancellationToken) + { + if (!vfs.FileExists(virtualPath)) + { + return null; + } + + try + { + await using var stream = await vfs.OpenReadAsync(virtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return null; + } + + using var reader = new StreamReader(stream, PythonEncoding.Utf8, detectEncodingFromByteOrderMarks: true); + return await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + return null; + } + } + + private static async Task AppendDirectUrlAsync( + LanguageAnalyzerContext context, + PythonVirtualFileSystem vfs, + ICollection> metadata, + ICollection evidence, + string directUrlVirtualPath, + CancellationToken cancellationToken) + { + var file = vfs.GetFile(directUrlVirtualPath); + if (file is null) + { + return; + } + + try + { + await using var stream = await vfs.OpenReadAsync(directUrlVirtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return; + } + + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + + var url = root.TryGetProperty("url", out var urlElement) ? urlElement.GetString() : null; + var isEditable = root.TryGetProperty("dir_info", out var dirInfo) && + dirInfo.TryGetProperty("editable", out var editableValue) && + editableValue.GetBoolean(); + var subdir = root.TryGetProperty("dir_info", out dirInfo) && + dirInfo.TryGetProperty("subdirectory", out var subdirElement) + ? subdirElement.GetString() + : null; + + string? vcs = null; + string? commit = null; + + if (root.TryGetProperty("vcs_info", out var vcsInfo)) + { + vcs = vcsInfo.TryGetProperty("vcs", out var vcsElement) ? vcsElement.GetString() : null; + commit = vcsInfo.TryGetProperty("commit_id", out var commitElement) ? commitElement.GetString() : null; + } + + if (isEditable) + { + AppendMetadata(metadata, "editable", "true"); + } + + AppendMetadata(metadata, "sourceUrl", url); + AppendMetadata(metadata, "sourceSubdirectory", subdir); + AppendMetadata(metadata, "sourceVcs", vcs); + AppendMetadata(metadata, "sourceCommit", commit); + + if (!string.IsNullOrWhiteSpace(url)) + { + var locator = file.IsFromArchive && file.ArchivePath is not null + ? PythonPathHelper.NormalizeRelative(context, file.ArchivePath) + : PythonPathHelper.NormalizeRelative(context, file.AbsolutePath); + + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.Metadata, + "direct_url.json", + locator, + url, + Sha256: null)); + } + } + catch (JsonException) + { + // Ignore invalid JSON + } + catch (IOException) + { + // Ignore read errors + } + } + + private static async Task> ReadRecordAsync( + PythonVirtualFileSystem vfs, + string recordVirtualPath, + CancellationToken cancellationToken) + { + if (!vfs.FileExists(recordVirtualPath)) + { + return Array.Empty(); + } + + var fileName = Path.GetFileName(recordVirtualPath); + if (!string.IsNullOrWhiteSpace(fileName) && + fileName.EndsWith("installed-files.txt", StringComparison.OrdinalIgnoreCase)) + { + return await ReadInstalledFilesAsync(vfs, recordVirtualPath, cancellationToken).ConfigureAwait(false); + } + + try + { + await using var stream = await vfs.OpenReadAsync(recordVirtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return Array.Empty(); + } + + using var reader = new StreamReader(stream, PythonEncoding.Utf8, detectEncodingFromByteOrderMarks: true); + var entries = new List(); + + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (line.Length == 0) + { + continue; + } + + var fields = ParseCsvLine(line); + if (fields.Count < 1) + { + continue; + } + + var entryPath = fields[0]; + string? algorithm = null; + string? hashValue = null; + + if (fields.Count > 1 && !string.IsNullOrWhiteSpace(fields[1])) + { + var hashField = fields[1].Trim(); + var separator = hashField.IndexOf('='); + if (separator > 0 && separator < hashField.Length - 1) + { + algorithm = hashField[..separator]; + hashValue = hashField[(separator + 1)..]; + } + } + + long? size = null; + if (fields.Count > 2 && + long.TryParse(fields[2], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedSize)) + { + size = parsedSize; + } + + entries.Add(new PythonRecordEntry(entryPath, algorithm, hashValue, size)); + } + + return entries; + } + catch (IOException) + { + return Array.Empty(); + } + } + + private static async Task> ReadInstalledFilesAsync( + PythonVirtualFileSystem vfs, + string recordVirtualPath, + CancellationToken cancellationToken) + { + try + { + await using var stream = await vfs.OpenReadAsync(recordVirtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return Array.Empty(); + } + + using var reader = new StreamReader(stream, PythonEncoding.Utf8, detectEncodingFromByteOrderMarks: true); + var entries = new List(); + + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + var trimmed = line.Trim(); + if (trimmed.Length == 0 || trimmed == ".") + { + continue; + } + + entries.Add(new PythonRecordEntry(trimmed, null, null, null)); + } + + return entries; + } + catch (IOException) + { + return Array.Empty(); + } + } + + private static async Task VerifyRecordAsync( + PythonVirtualFileSystem vfs, + string distInfoVirtualPath, + IReadOnlyList entries, + CancellationToken cancellationToken) + { + if (entries.Count == 0) + { + return new PythonRecordVerificationResult(0, 0, 0, 0, 0, usedByEntrypoint: false, Array.Empty(), Array.Empty()); + } + + var evidence = new List(); + var unsupported = new HashSet(StringComparer.OrdinalIgnoreCase); + var root = GetParentDirectory(distInfoVirtualPath); + + var total = 0; + var hashed = 0; + var missing = 0; + var mismatched = 0; + var ioErrors = 0; + + foreach (var entry in entries) + { + cancellationToken.ThrowIfCancellationRequested(); + total++; + + var normalizedEntryPath = NormalizeRecordPath(entry.Path); + if (normalizedEntryPath is null) + { + missing++; + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.Derived, + "RECORD", + NormalizeVfsPath(entry.Path), + "outside-root", + Sha256: null)); + continue; + } + + var virtualPath = $"{root}/{normalizedEntryPath}"; + + if (!vfs.FileExists(virtualPath)) + { + missing++; + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.Derived, + "RECORD", + NormalizeVfsPath(virtualPath), + "missing", + Sha256: null)); + continue; + } + + if (string.IsNullOrWhiteSpace(entry.HashAlgorithm) || string.IsNullOrWhiteSpace(entry.HashValue)) + { + continue; + } + + hashed++; + + if (!string.Equals(entry.HashAlgorithm, "sha256", StringComparison.OrdinalIgnoreCase)) + { + unsupported.Add(entry.HashAlgorithm); + continue; + } + + string? actualHash; + try + { + actualHash = await ComputeSha256Base64Async(vfs, virtualPath, cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + ioErrors++; + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.Derived, + "RECORD", + NormalizeVfsPath(virtualPath), + "io-error", + Sha256: null)); + continue; + } + + if (!string.Equals(actualHash, entry.HashValue, StringComparison.Ordinal)) + { + mismatched++; + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.Derived, + "RECORD", + NormalizeVfsPath(virtualPath), + $"sha256 mismatch expected={entry.HashValue} actual={actualHash}", + Sha256: actualHash)); + } + } + + return new PythonRecordVerificationResult( + total, + hashed, + missing, + mismatched, + ioErrors, + usedByEntrypoint: false, + unsupported.ToArray(), + evidence); + } + + private static async Task ComputeSha256Base64Async( + PythonVirtualFileSystem vfs, + string virtualPath, + CancellationToken cancellationToken) + { + await using var stream = await vfs.OpenReadAsync(virtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + throw new IOException("Unable to open file for hashing."); + } + + using var sha = SHA256.Create(); + var buffer = ArrayPool.Shared.Rent(81920); + try + { + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0) + { + sha.TransformBlock(buffer, 0, bytesRead, null, 0); + } + + sha.TransformFinalBlock(Array.Empty(), 0, 0); + return Convert.ToBase64String(sha.Hash ?? Array.Empty()); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static List ParseCsvLine(string line) + { + var values = new List(); + var builder = new StringBuilder(); + var inQuotes = false; + + for (var i = 0; i < line.Length; i++) + { + var ch = line[i]; + + if (inQuotes) + { + if (ch == '"') + { + var next = i + 1 < line.Length ? line[i + 1] : '\0'; + if (next == '"') + { + builder.Append('"'); + i++; + } + else + { + inQuotes = false; + } + } + else + { + builder.Append(ch); + } + + continue; + } + + if (ch == ',') + { + values.Add(builder.ToString()); + builder.Clear(); + continue; + } + + if (ch == '"') + { + inQuotes = true; + continue; + } + + builder.Append(ch); + } + + values.Add(builder.ToString()); + return values; + } + + private static string? NormalizeRecordPath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return null; + } + + var normalized = path.Replace('\\', '/').TrimStart('/'); + + if (normalized.Contains("/../", StringComparison.Ordinal) || + normalized.StartsWith("../", StringComparison.Ordinal) || + normalized.EndsWith("/..", StringComparison.Ordinal) || + normalized == "..") + { + return null; + } + + return normalized.Length == 0 ? null : normalized; + } + + private static string NormalizeVfsPath(string path) + => path.Replace('\\', '/').Trim('/'); + + private static string GetParentDirectory(string path) + { + var normalized = NormalizeVfsPath(path); + var lastSlash = normalized.LastIndexOf('/'); + return lastSlash <= 0 ? string.Empty : normalized[..lastSlash]; + } + + private static void AppendMetadata(ICollection> metadata, string key, string? value) + { + if (string.IsNullOrWhiteSpace(key)) + { + return; + } + + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + metadata.Add(new KeyValuePair(key, value.Trim())); + } + + private sealed class PythonMetadataDocumentVfs + { + private readonly Dictionary> _values; + + private PythonMetadataDocumentVfs(Dictionary> values) + { + _values = values; + } + + public static async Task LoadAsync( + PythonVirtualFileSystem vfs, + string virtualPath, + CancellationToken cancellationToken) + { + if (!vfs.FileExists(virtualPath)) + { + return new PythonMetadataDocumentVfs(new Dictionary>(StringComparer.OrdinalIgnoreCase)); + } + + try + { + await using var stream = await vfs.OpenReadAsync(virtualPath, cancellationToken).ConfigureAwait(false); + if (stream is null) + { + return new PythonMetadataDocumentVfs(new Dictionary>(StringComparer.OrdinalIgnoreCase)); + } + + using var reader = new StreamReader(stream, PythonEncoding.Utf8, detectEncodingFromByteOrderMarks: true); + var values = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + string? currentKey = null; + var builder = new StringBuilder(); + + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (line.Length == 0) + { + Commit(); + continue; + } + + if (line.StartsWith(' ') || line.StartsWith('\t')) + { + if (currentKey is not null) + { + if (builder.Length > 0) + { + builder.Append(' '); + } + + builder.Append(line.Trim()); + } + + continue; + } + + Commit(); + + var separator = line.IndexOf(':'); + if (separator <= 0) + { + continue; + } + + currentKey = line[..separator].Trim(); + builder.Clear(); + builder.Append(line[(separator + 1)..].Trim()); + } + + Commit(); + return new PythonMetadataDocumentVfs(values); + + void Commit() + { + if (string.IsNullOrWhiteSpace(currentKey)) + { + return; + } + + if (!values.TryGetValue(currentKey, out var list)) + { + list = new List(); + values[currentKey] = list; + } + + var value = builder.ToString().Trim(); + if (value.Length > 0) + { + list.Add(value); + } + + currentKey = null; + builder.Clear(); + } + } + catch (IOException) + { + return new PythonMetadataDocumentVfs(new Dictionary>(StringComparer.OrdinalIgnoreCase)); + } + } + + public string? GetFirst(string key) + { + if (key is null) + { + return null; + } + + return _values.TryGetValue(key, out var list) && list.Count > 0 + ? list[0] + : null; + } + + public IReadOnlyList GetAll(string key) + { + if (key is null) + { + return Array.Empty(); + } + + return _values.TryGetValue(key, out var list) + ? list.AsReadOnly() + : Array.Empty(); + } + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonInputNormalizer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonInputNormalizer.cs index 826b2fe0b..098f37477 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonInputNormalizer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonInputNormalizer.cs @@ -1,6 +1,7 @@ using System.Text.Json; using System.Text.RegularExpressions; using System.Linq; +using StellaOps.Scanner.Analyzers.Lang.Python.Internal; namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem; @@ -81,9 +82,11 @@ internal sealed partial class PythonInputNormalizer await DetectLayoutAsync(cancellationToken).ConfigureAwait(false); await DetectVersionTargetsAsync(cancellationToken).ConfigureAwait(false); DetectSitePackages(); + DetectLayerSitePackages(); DetectWheels(); DetectZipapps(); await DetectEditablesAsync(cancellationToken).ConfigureAwait(false); + NormalizeDetectedInputs(); return this; } @@ -94,6 +97,11 @@ internal sealed partial class PythonInputNormalizer { var builder = PythonVirtualFileSystem.CreateBuilder(); + AddProjectFiles(builder); + + var condaMeta = Path.Combine(_rootPath, "conda-meta"); + builder.AddDirectory(condaMeta, "conda-meta", PythonFileSource.ProjectConfig, includeHiddenFiles: false); + // Add site-packages in order (later takes precedence) foreach (var sitePackagesPath in _sitePackagesPaths) { @@ -582,7 +590,8 @@ internal sealed partial class PythonInputNormalizer { try { - foreach (var pythonDir in Directory.EnumerateDirectories(libDir, "python*", SafeEnumeration)) + foreach (var pythonDir in Directory.EnumerateDirectories(libDir, "python*", SafeEnumeration) + .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) { searchPaths.Add(Path.Combine(pythonDir, "site-packages")); } @@ -611,6 +620,25 @@ internal sealed partial class PythonInputNormalizer searchPaths.Add(Path.Combine(_rootPath, "usr", "local", "lib", "python3.12", "site-packages")); searchPaths.Add(Path.Combine(_rootPath, "usr", "lib", "python3", "dist-packages")); + // System-style lib/pythonX.Y/site-packages under the workspace root + var rootLibDir = Path.Combine(_rootPath, "lib"); + if (Directory.Exists(rootLibDir)) + { + try + { + foreach (var pythonDir in Directory.EnumerateDirectories(rootLibDir, "python*", SafeEnumeration) + .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) + { + searchPaths.Add(Path.Combine(pythonDir, "site-packages")); + searchPaths.Add(Path.Combine(pythonDir, "dist-packages")); + } + } + catch (UnauthorizedAccessException) + { + // Ignore + } + } + // Root site-packages (common for some Docker images) searchPaths.Add(Path.Combine(_rootPath, "site-packages")); @@ -623,6 +651,17 @@ internal sealed partial class PythonInputNormalizer } } + private void DetectLayerSitePackages() + { + foreach (var sitePackagesPath in PythonContainerAdapter.DiscoverLayerSitePackages(_rootPath)) + { + if (!_sitePackagesPaths.Contains(sitePackagesPath, StringComparer.OrdinalIgnoreCase)) + { + _sitePackagesPaths.Add(sitePackagesPath); + } + } + } + private void DetectWheels() { // Look for wheels in common locations @@ -643,7 +682,8 @@ internal sealed partial class PythonInputNormalizer try { - foreach (var wheel in Directory.EnumerateFiles(searchPath, "*.whl", SafeEnumeration)) + foreach (var wheel in Directory.EnumerateFiles(searchPath, "*.whl", SafeEnumeration) + .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) { if (!_wheelPaths.Contains(wheel, StringComparer.OrdinalIgnoreCase)) { @@ -700,37 +740,24 @@ internal sealed partial class PythonInputNormalizer private void DetectZipapps() { - if (!Directory.Exists(_rootPath)) + foreach (var zipappPath in PythonZipappAdapter.DiscoverZipapps(_rootPath)) { - return; - } - - try - { - foreach (var pyz in Directory.EnumerateFiles(_rootPath, "*.pyz", SafeEnumeration)) + if (!_zipappPaths.Contains(zipappPath, StringComparer.OrdinalIgnoreCase)) { - _zipappPaths.Add(pyz); + _zipappPaths.Add(zipappPath); } - - foreach (var pyzw in Directory.EnumerateFiles(_rootPath, "*.pyzw", SafeEnumeration)) - { - _zipappPaths.Add(pyzw); - } - } - catch (UnauthorizedAccessException) - { - // Ignore } } private async Task DetectEditablesAsync(CancellationToken cancellationToken) { // Look for .egg-link files in site-packages - foreach (var sitePackagesPath in _sitePackagesPaths) + foreach (var sitePackagesPath in _sitePackagesPaths.OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) { try { - foreach (var eggLink in Directory.EnumerateFiles(sitePackagesPath, "*.egg-link", SafeEnumeration)) + foreach (var eggLink in Directory.EnumerateFiles(sitePackagesPath, "*.egg-link", SafeEnumeration) + .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) { var content = await File.ReadAllTextAsync(eggLink, cancellationToken).ConfigureAwait(false); var lines = content.Split('\n', StringSplitOptions.RemoveEmptyEntries); @@ -761,11 +788,12 @@ internal sealed partial class PythonInputNormalizer } // Look for direct_url.json with editable flag in dist-info directories - foreach (var sitePackagesPath in _sitePackagesPaths) + foreach (var sitePackagesPath in _sitePackagesPaths.OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) { try { - foreach (var distInfo in Directory.EnumerateDirectories(sitePackagesPath, "*.dist-info", SafeEnumeration)) + foreach (var distInfo in Directory.EnumerateDirectories(sitePackagesPath, "*.dist-info", SafeEnumeration) + .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase)) { var directUrlPath = Path.Combine(distInfo, "direct_url.json"); if (!File.Exists(directUrlPath)) @@ -815,6 +843,84 @@ internal sealed partial class PythonInputNormalizer } } + private void NormalizeDetectedInputs() + { + NormalizePathList(_sitePackagesPaths); + NormalizePathList(_wheelPaths); + NormalizePathList(_zipappPaths); + NormalizeEditableList(_editablePaths); + } + + private static void NormalizePathList(List paths) + { + var normalized = paths + .Where(static p => !string.IsNullOrWhiteSpace(p)) + .Select(static p => Path.GetFullPath(p.Trim())) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static p => p, StringComparer.Ordinal) + .ToList(); + + paths.Clear(); + paths.AddRange(normalized); + } + + private static void NormalizeEditableList(List<(string Path, string? PackageName)> editables) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var normalized = new List<(string Path, string? PackageName)>(); + + foreach (var (path, packageName) in editables) + { + if (string.IsNullOrWhiteSpace(path)) + { + continue; + } + + var fullPath = Path.GetFullPath(path.Trim()); + var name = string.IsNullOrWhiteSpace(packageName) ? null : packageName.Trim(); + var key = $"{name ?? string.Empty}|{fullPath}"; + + if (!seen.Add(key)) + { + continue; + } + + normalized.Add((fullPath, name)); + } + + editables.Clear(); + editables.AddRange(normalized + .OrderBy(static e => e.PackageName ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static e => e.Path, StringComparer.Ordinal)); + } + + private void AddProjectFiles(PythonVirtualFileSystem.Builder builder) + { + AddProjectFile(builder, "pyproject.toml", PythonFileSource.ProjectConfig); + AddProjectFile(builder, "setup.py", PythonFileSource.ProjectConfig); + AddProjectFile(builder, "setup.cfg", PythonFileSource.ProjectConfig); + AddProjectFile(builder, "runtime.txt", PythonFileSource.ProjectConfig); + AddProjectFile(builder, "Dockerfile", PythonFileSource.ProjectConfig); + AddProjectFile(builder, "tox.ini", PythonFileSource.ProjectConfig); + + AddProjectFile(builder, "requirements.txt", PythonFileSource.LockFile); + AddProjectFile(builder, "requirements-dev.txt", PythonFileSource.LockFile); + AddProjectFile(builder, "requirements.prod.txt", PythonFileSource.LockFile); + AddProjectFile(builder, "Pipfile.lock", PythonFileSource.LockFile); + AddProjectFile(builder, "poetry.lock", PythonFileSource.LockFile); + } + + private void AddProjectFile(PythonVirtualFileSystem.Builder builder, string relativePath, PythonFileSource source) + { + var absolutePath = Path.Combine(_rootPath, relativePath); + if (!File.Exists(absolutePath)) + { + return; + } + + builder.AddFile(relativePath, absolutePath, source); + } + [GeneratedRegex(@"requires-python\s*=\s*[""']?(?[^""'\n]+)", RegexOptions.IgnoreCase)] private static partial Regex RequiresPythonPattern(); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonVirtualFileSystem.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonVirtualFileSystem.cs index 459a4c941..5064dc9d8 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonVirtualFileSystem.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/VirtualFileSystem/PythonVirtualFileSystem.cs @@ -54,9 +54,9 @@ internal sealed partial class PythonVirtualFileSystem public int FileCount => _files.Count; /// - /// Gets all files in the virtual filesystem. + /// Gets all files in the virtual filesystem, ordered deterministically by virtual path. /// - public IEnumerable Files => _files.Values; + public IEnumerable Files => Paths.Select(path => _files[path]); /// /// Gets all virtual paths in sorted order. @@ -230,17 +230,17 @@ internal sealed partial class PythonVirtualFileSystem var normalized = NormalizePath(virtualPath); var prefix = normalized.Length == 0 ? string.Empty : normalized + "/"; - foreach (var kvp in _files) + foreach (var key in _files.Keys.OrderBy(static path => path, StringComparer.Ordinal)) { - if (!kvp.Key.StartsWith(prefix, StringComparison.Ordinal)) + if (!key.StartsWith(prefix, StringComparison.Ordinal)) { continue; } - var relative = kvp.Key[prefix.Length..]; + var relative = key[prefix.Length..]; if (regex.IsMatch(relative)) { - yield return kvp.Value; + yield return _files[key]; } } } @@ -291,11 +291,32 @@ internal sealed partial class PythonVirtualFileSystem { private readonly Dictionary _files = new(StringComparer.Ordinal); private readonly HashSet _processedArchives = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _archiveAliasCounters = new(StringComparer.OrdinalIgnoreCase); private readonly HashSet _sourceTreeRoots = new(StringComparer.Ordinal); private readonly HashSet _sitePackagesPaths = new(StringComparer.Ordinal); private readonly HashSet _editablePaths = new(StringComparer.Ordinal); private readonly HashSet _zipArchivePaths = new(StringComparer.Ordinal); + /// + /// Adds files from an arbitrary directory at a specific virtual prefix. + /// + public Builder AddDirectory( + string directoryPath, + string virtualPrefix, + PythonFileSource source, + string? layerDigest = null, + bool includeHiddenFiles = false) + { + if (!Directory.Exists(directoryPath)) + { + return this; + } + + var basePath = Path.GetFullPath(directoryPath); + AddDirectoryRecursive(basePath, NormalizePath(virtualPrefix), source, layerDigest, includeHiddenFiles); + return this; + } + /// /// Adds files from a site-packages directory. /// @@ -308,7 +329,7 @@ internal sealed partial class PythonVirtualFileSystem var basePath = Path.GetFullPath(sitePackagesPath); _sitePackagesPaths.Add(string.Empty); // Root of the VFS - AddDirectoryRecursive(basePath, string.Empty, PythonFileSource.SitePackages, layerDigest); + AddDirectoryRecursive(basePath, string.Empty, PythonFileSource.SitePackages, layerDigest, includeHiddenFiles: false); return this; } @@ -322,12 +343,13 @@ internal sealed partial class PythonVirtualFileSystem return this; } - _zipArchivePaths.Add(wheelPath); + var virtualRoot = CreateArchiveVirtualRoot("wheel", wheelPath); + _zipArchivePaths.Add(virtualRoot); try { using var archive = ZipFile.OpenRead(wheelPath); - AddArchiveEntries(archive, wheelPath, PythonFileSource.Wheel); + AddArchiveEntries(archive, wheelPath, virtualRoot, PythonFileSource.Wheel); } catch (InvalidDataException) { @@ -351,7 +373,8 @@ internal sealed partial class PythonVirtualFileSystem return this; } - _zipArchivePaths.Add(zipappPath); + var virtualRoot = CreateArchiveVirtualRoot("zipapp", zipappPath); + _zipArchivePaths.Add(virtualRoot); try { @@ -366,7 +389,7 @@ internal sealed partial class PythonVirtualFileSystem stream.Position = offset; using var archive = new ZipArchive(stream, ZipArchiveMode.Read); - AddArchiveEntries(archive, zipappPath, PythonFileSource.Zipapp); + AddArchiveEntries(archive, zipappPath, virtualRoot, PythonFileSource.Zipapp); } catch (InvalidDataException) { @@ -390,14 +413,15 @@ internal sealed partial class PythonVirtualFileSystem return this; } - _zipArchivePaths.Add(sdistPath); + var virtualRoot = CreateArchiveVirtualRoot("sdist", sdistPath); + _zipArchivePaths.Add(virtualRoot); try { if (sdistPath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) { using var archive = ZipFile.OpenRead(sdistPath); - AddArchiveEntries(archive, sdistPath, PythonFileSource.Sdist); + AddArchiveEntries(archive, sdistPath, virtualRoot, PythonFileSource.Sdist); } // Note: .tar.gz support would require TarReader from System.Formats.Tar // For now, we handle the common .zip case @@ -427,7 +451,7 @@ internal sealed partial class PythonVirtualFileSystem var basePath = Path.GetFullPath(editablePath); var prefix = string.IsNullOrEmpty(packageName) ? string.Empty : packageName + "/"; _editablePaths.Add(prefix.TrimEnd('/')); - AddDirectoryRecursive(basePath, prefix.TrimEnd('/'), PythonFileSource.Editable, layerDigest: null); + AddDirectoryRecursive(basePath, prefix.TrimEnd('/'), PythonFileSource.Editable, layerDigest: null, includeHiddenFiles: false); return this; } @@ -443,7 +467,7 @@ internal sealed partial class PythonVirtualFileSystem var basePath = Path.GetFullPath(sourcePath); _sourceTreeRoots.Add(string.Empty); // Root of the VFS - AddDirectoryRecursive(basePath, string.Empty, PythonFileSource.SourceTree, layerDigest: null); + AddDirectoryRecursive(basePath, string.Empty, PythonFileSource.SourceTree, layerDigest: null, includeHiddenFiles: false); return this; } @@ -522,7 +546,8 @@ internal sealed partial class PythonVirtualFileSystem string basePath, string virtualPrefix, PythonFileSource source, - string? layerDigest) + string? layerDigest, + bool includeHiddenFiles) { try { @@ -537,7 +562,7 @@ internal sealed partial class PythonVirtualFileSystem // Skip __pycache__ and hidden files if (normalizedRelative.Contains("/__pycache__/", StringComparison.Ordinal) || normalizedRelative.StartsWith("__pycache__/", StringComparison.Ordinal) || - Path.GetFileName(file).StartsWith('.')) + (!includeHiddenFiles && Path.GetFileName(file).StartsWith('.'))) { continue; } @@ -566,7 +591,7 @@ internal sealed partial class PythonVirtualFileSystem } } - private void AddArchiveEntries(ZipArchive archive, string archivePath, PythonFileSource source) + private void AddArchiveEntries(ZipArchive archive, string archivePath, string virtualRoot, PythonFileSource source) { foreach (var entry in archive.Entries) { @@ -576,7 +601,8 @@ internal sealed partial class PythonVirtualFileSystem continue; } - var virtualPath = entry.FullName.Replace('\\', '/'); + var entryPath = entry.FullName.Replace('\\', '/').TrimStart('/'); + var virtualPath = $"{virtualRoot}/{entryPath}"; // Skip __pycache__ in archives too if (virtualPath.Contains("/__pycache__/", StringComparison.Ordinal) || @@ -587,7 +613,7 @@ internal sealed partial class PythonVirtualFileSystem AddFile( virtualPath, - entry.FullName, + entryPath, source, layerDigest: null, archivePath: archivePath, @@ -595,6 +621,22 @@ internal sealed partial class PythonVirtualFileSystem } } + private string CreateArchiveVirtualRoot(string kind, string archivePath) + { + var baseName = Path.GetFileName(archivePath); + var key = $"{kind}/{baseName}"; + + if (!_archiveAliasCounters.TryGetValue(key, out var count)) + { + _archiveAliasCounters[key] = 1; + return $"archives/{kind}/{baseName}"; + } + + count++; + _archiveAliasCounters[key] = count; + return $"archives/{kind}/{baseName}~{count}"; + } + private static long FindZipOffset(Stream stream) { // ZIP files start with PK\x03\x04 signature diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/PythonLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/PythonLanguageAnalyzer.cs index 6ccd156ae..30a3f620e 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/PythonLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/PythonLanguageAnalyzer.cs @@ -1,18 +1,13 @@ using System.Linq; using System.Text.Json; using StellaOps.Scanner.Analyzers.Lang.Python.Internal; +using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Packaging; +using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem; namespace StellaOps.Scanner.Analyzers.Lang.Python; public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer { - private static readonly EnumerationOptions Enumeration = new() - { - RecurseSubdirectories = true, - IgnoreInaccessible = true, - AttributesToSkip = FileAttributes.Device | FileAttributes.ReparsePoint - }; - public string Id => "python"; public string DisplayName => "Python Analyzer"; @@ -43,73 +38,33 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer // Analyze zipapps in workspace and container layers var zipappAnalysis = PythonZipappAdapter.AnalyzeAll(context.RootPath); - // Collect dist-info directories from both root and container layers - var distInfoDirectories = CollectDistInfoDirectories(context.RootPath); + var projectAnalysis = await PythonProjectAnalysis.AnalyzeAsync(context.RootPath, cancellationToken).ConfigureAwait(false); + var vfs = projectAnalysis.VirtualFileSystem; - foreach (var distInfoPath in distInfoDirectories) + var packageDiscovery = new PythonPackageDiscovery(); + var discoveryResult = await packageDiscovery.DiscoverAsync(vfs, cancellationToken).ConfigureAwait(false); + + foreach (var package in discoveryResult.Packages + .Where(static p => !string.IsNullOrWhiteSpace(p.Version)) + .OrderBy(static p => p.NormalizedName, StringComparer.Ordinal) + .ThenBy(static p => p.Version, StringComparer.Ordinal)) { cancellationToken.ThrowIfCancellationRequested(); - PythonDistribution? distribution; - try - { - distribution = await PythonDistributionLoader.LoadAsync(context, distInfoPath, cancellationToken).ConfigureAwait(false); - } - catch (IOException) - { - continue; - } - catch (JsonException) - { - continue; - } - catch (UnauthorizedAccessException) - { - continue; - } - - if (distribution is null) - { - continue; - } - - var metadata = distribution.SortedMetadata.ToList(); - - if (lockData.TryGet(distribution.Name, distribution.Version, out var lockEntry)) - { - matchedLocks.Add(lockEntry!.DeclarationKey); - AppendLockMetadata(metadata, lockEntry); - } - else if (hasLockEntries) - { - metadata.Add(new KeyValuePair("lockMissing", "true")); - } - - // Append runtime information - AppendRuntimeMetadata(metadata, runtimeInfo); - - // Append environment variables (PYTHONPATH/PYTHONHOME) - AppendEnvironmentMetadata(metadata, environment); - - // Append startup hooks warnings - AppendStartupHooksMetadata(metadata, startupHooks); - - // Append zipapp analysis - AppendZipappMetadata(metadata, zipappAnalysis); - - // Collect evidence including startup hooks - var evidence = distribution.SortedEvidence.ToList(); - evidence.AddRange(startupHooks.ToEvidence(context)); - - writer.AddFromPurl( - analyzerId: "python", - purl: distribution.Purl, - name: distribution.Name, - version: distribution.Version, - type: "pypi", - metadata: metadata, - evidence: evidence, - usedByEntrypoint: distribution.UsedByEntrypoint); + await EmitDiscoveredPackageAsync( + context, + writer, + vfs, + package, + lockData, + matchedLocks, + hasLockEntries, + runtimeInfo, + environment, + startupHooks, + zipappAnalysis, + cancellationToken) + .ConfigureAwait(false); } if (lockData.Entries.Count > 0) @@ -121,18 +76,18 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer continue; } + var normalizedName = PythonPathHelper.NormalizePackageName(entry.Name); + var declaredMetadata = new List> { new("declaredOnly", "true"), + new("pkg.kind", "DeclaredOnly"), + new("pkg.confidence", PythonPackageConfidence.Medium.ToString()), + new("pkg.location", entry.Locator), new("lockSource", entry.Source), new("lockLocator", entry.Locator) }; - AppendCommonLockFields(declaredMetadata, entry); - - var version = string.IsNullOrWhiteSpace(entry.Version) ? "editable" : entry.Version!; - var purl = $"pkg:pypi/{PythonPathHelper.NormalizePackageName(entry.Name)}@{version}"; - var evidence = new[] { new LanguageComponentEvidence( @@ -143,6 +98,49 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer Sha256: null) }; + if (string.IsNullOrWhiteSpace(entry.Version)) + { + var editableSpec = NormalizeEditableSpec(context, entry.EditablePath, out var specRedacted); + + declaredMetadata.Add(new KeyValuePair("declared.source", entry.Source)); + declaredMetadata.Add(new KeyValuePair("declared.locator", entry.Locator)); + declaredMetadata.Add(new KeyValuePair("declared.versionSpec", editableSpec)); + declaredMetadata.Add(new KeyValuePair("declared.scope", "unknown")); + declaredMetadata.Add(new KeyValuePair("declared.sourceType", "editable")); + + if (!string.IsNullOrWhiteSpace(editableSpec)) + { + declaredMetadata.Add(new KeyValuePair("lockEditablePath", editableSpec)); + } + + if (specRedacted) + { + declaredMetadata.Add(new KeyValuePair("lockEditablePathRedacted", "true")); + } + + var componentKey = LanguageExplicitKey.Create("python", "pypi", normalizedName, editableSpec, entry.Locator); + writer.AddFromExplicitKey( + analyzerId: "python", + componentKey: componentKey, + purl: null, + name: entry.Name, + version: null, + type: "pypi", + metadata: declaredMetadata, + evidence: evidence, + usedByEntrypoint: false); + continue; + } + + AppendCommonLockFields(declaredMetadata, entry); + + var version = entry.Version!.Trim(); + if (version.Length == 0) + { + continue; + } + + var purl = $"pkg:pypi/{normalizedName}@{version}"; writer.AddFromPurl( analyzerId: "python", purl: purl, @@ -156,6 +154,284 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer } } + private static string NormalizeEditableSpec(LanguageAnalyzerContext context, string? editablePath, out bool redacted) + { + redacted = false; + + if (string.IsNullOrWhiteSpace(editablePath)) + { + return string.Empty; + } + + var trimmed = editablePath.Trim().Trim('"', '\''); + if (trimmed.Length == 0) + { + return string.Empty; + } + + var normalized = trimmed.Replace('\\', '/'); + var hasDrivePrefix = trimmed.Length >= 2 && char.IsLetter(trimmed[0]) && trimmed[1] == ':'; + var isAbsolute = Path.IsPathRooted(trimmed) || + hasDrivePrefix || + normalized.StartsWith("/", StringComparison.Ordinal) || + normalized.StartsWith("//", StringComparison.Ordinal); + + if (!isAbsolute) + { + return normalized; + } + + try + { + var relative = context.GetRelativePath(trimmed); + if (!string.IsNullOrWhiteSpace(relative) && + relative != "." && + !relative.StartsWith("..", StringComparison.Ordinal) && + relative.IndexOf(':') < 0) + { + return relative.Replace('\\', '/'); + } + } + catch + { + } + + redacted = true; + normalized = normalized.TrimEnd('/'); + var lastSlash = normalized.LastIndexOf('/'); + var fileName = lastSlash >= 0 && lastSlash < normalized.Length - 1 ? normalized[(lastSlash + 1)..] : normalized; + return string.IsNullOrWhiteSpace(fileName) ? "editable" : fileName; + } + + private static async Task EmitDiscoveredPackageAsync( + LanguageAnalyzerContext context, + LanguageComponentWriter writer, + PythonVirtualFileSystem vfs, + PythonPackageInfo package, + PythonLockData lockData, + ISet matchedLocks, + bool hasLockEntries, + PythonRuntimeInfo? runtimeInfo, + PythonEnvironment environment, + PythonStartupHooks startupHooks, + PythonZipappAnalysis zipappAnalysis, + CancellationToken cancellationToken) + { + var version = package.Version!.Trim(); + if (version.Length == 0) + { + return; + } + + var metadata = new List>(); + metadata.AddRange(BuildPackageMetadata(context, vfs, package)); + + if (lockData.TryGet(package.Name, version, out var lockEntry)) + { + matchedLocks.Add(lockEntry!.DeclarationKey); + AppendLockMetadata(metadata, lockEntry); + } + else if (hasLockEntries) + { + metadata.Add(new KeyValuePair("lockMissing", "true")); + } + + var metadataDirectory = TryResolvePhysicalMetadataDirectory(vfs, package, out var metadataFile); + if (metadataDirectory is not null) + { + PythonDistribution? distribution; + try + { + distribution = await PythonDistributionLoader.LoadAsync(context, metadataDirectory, cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + return; + } + catch (JsonException) + { + return; + } + catch (UnauthorizedAccessException) + { + return; + } + + if (distribution is null) + { + return; + } + + var fullMetadata = distribution.SortedMetadata.ToList(); + fullMetadata.AddRange(metadata); + + AppendRuntimeMetadata(fullMetadata, runtimeInfo); + AppendEnvironmentMetadata(fullMetadata, environment); + AppendStartupHooksMetadata(fullMetadata, startupHooks); + AppendZipappMetadata(fullMetadata, zipappAnalysis); + + var evidence = distribution.SortedEvidence.ToList(); + evidence.AddRange(startupHooks.ToEvidence(context)); + + writer.AddFromPurl( + analyzerId: "python", + purl: distribution.Purl, + name: distribution.Name, + version: distribution.Version, + type: "pypi", + metadata: fullMetadata, + evidence: evidence, + usedByEntrypoint: distribution.UsedByEntrypoint); + + return; + } + + if (metadataFile is not null && metadataFile.IsFromArchive) + { + var archiveDistribution = await PythonDistributionVfsLoader + .LoadAsync(context, vfs, package, cancellationToken) + .ConfigureAwait(false); + + if (archiveDistribution is not null) + { + var fullMetadata = archiveDistribution.SortedMetadata.ToList(); + fullMetadata.AddRange(metadata); + + writer.AddFromPurl( + analyzerId: "python", + purl: archiveDistribution.Purl, + name: archiveDistribution.Name, + version: archiveDistribution.Version, + type: "pypi", + metadata: fullMetadata, + evidence: archiveDistribution.SortedEvidence, + usedByEntrypoint: archiveDistribution.UsedByEntrypoint); + + return; + } + } + + var purl = $"pkg:pypi/{PythonPathHelper.NormalizePackageName(package.Name)}@{version}"; + var evidenceFallback = BuildPackageEvidence(context, vfs, package, metadataFile); + + writer.AddFromPurl( + analyzerId: "python", + purl: purl, + name: package.Name, + version: version, + type: "pypi", + metadata: metadata, + evidence: evidenceFallback, + usedByEntrypoint: false); + } + + private static string? TryResolvePhysicalMetadataDirectory( + PythonVirtualFileSystem vfs, + PythonPackageInfo package, + out PythonVirtualFile? metadataFile) + { + metadataFile = null; + + if (string.IsNullOrWhiteSpace(package.MetadataPath)) + { + return null; + } + + var metadataName = package.Kind == PythonPackageKind.Egg ? "PKG-INFO" : "METADATA"; + var virtualPath = $"{package.MetadataPath}/{metadataName}"; + metadataFile = vfs.GetFile(virtualPath); + + if (metadataFile is null || metadataFile.IsFromArchive) + { + return null; + } + + return Path.GetDirectoryName(metadataFile.AbsolutePath); + } + + private static IEnumerable> BuildPackageMetadata( + LanguageAnalyzerContext context, + PythonVirtualFileSystem vfs, + PythonPackageInfo package) + { + var location = package.Location; + if (string.IsNullOrWhiteSpace(location) && !string.IsNullOrWhiteSpace(package.MetadataPath)) + { + var metadataName = package.Kind == PythonPackageKind.Egg ? "PKG-INFO" : "METADATA"; + var file = vfs.GetFile($"{package.MetadataPath}/{metadataName}"); + + if (file is not null) + { + if (file.IsFromArchive && file.ArchivePath is not null) + { + location = PythonPathHelper.NormalizeRelative(context, file.ArchivePath); + } + else + { + location = Path.GetDirectoryName(file.AbsolutePath) is { Length: > 0 } metadataDirectory + ? PythonPathHelper.NormalizeRelative(context, metadataDirectory) + : PythonPathHelper.NormalizeRelative(context, file.AbsolutePath); + } + } + } + + yield return new KeyValuePair("pkg.kind", package.Kind.ToString()); + yield return new KeyValuePair("pkg.confidence", package.Confidence.ToString()); + yield return new KeyValuePair("pkg.location", string.IsNullOrWhiteSpace(location) ? "." : location.Replace('\\', '/')); + } + + private static IReadOnlyCollection BuildPackageEvidence( + LanguageAnalyzerContext context, + PythonVirtualFileSystem vfs, + PythonPackageInfo package, + PythonVirtualFile? metadataFile) + { + if (metadataFile is not null) + { + var locator = metadataFile.IsFromArchive && metadataFile.ArchivePath is not null + ? PythonPathHelper.NormalizeRelative(context, metadataFile.ArchivePath) + : PythonPathHelper.NormalizeRelative(context, metadataFile.AbsolutePath); + + var value = metadataFile.IsFromArchive ? metadataFile.AbsolutePath : null; + + return new[] + { + new LanguageComponentEvidence( + LanguageEvidenceKind.File, + package.Kind == PythonPackageKind.Egg ? "PKG-INFO" : "METADATA", + locator, + Value: value, + Sha256: null) + }; + } + + if (!string.IsNullOrWhiteSpace(package.MetadataPath)) + { + var metadataName = package.Kind == PythonPackageKind.Egg ? "PKG-INFO" : "METADATA"; + var file = vfs.GetFile($"{package.MetadataPath}/{metadataName}"); + if (file is not null) + { + var locator = file.IsFromArchive && file.ArchivePath is not null + ? PythonPathHelper.NormalizeRelative(context, file.ArchivePath) + : PythonPathHelper.NormalizeRelative(context, file.AbsolutePath); + + var value = file.IsFromArchive ? file.AbsolutePath : null; + + return new[] + { + new LanguageComponentEvidence( + LanguageEvidenceKind.File, + metadataName, + locator, + Value: value, + Sha256: null) + }; + } + } + + return Array.Empty(); + } + private static void AppendLockMetadata(List> metadata, PythonLockEntry entry) { metadata.Add(new KeyValuePair("lockSource", entry.Source)); @@ -286,41 +562,4 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer } } } - - private static IReadOnlyCollection CollectDistInfoDirectories(string rootPath) - { - var directories = new HashSet(StringComparer.OrdinalIgnoreCase); - - AddMetadataDirectories(rootPath, "*.dist-info", directories); - AddMetadataDirectories(rootPath, "*.egg-info", directories); - - // Also collect from OCI container layers - foreach (var dir in PythonContainerAdapter.DiscoverDistInfoDirectories(rootPath)) - { - directories.Add(dir); - } - - return directories - .OrderBy(static path => path, StringComparer.Ordinal) - .ToArray(); - - static void AddMetadataDirectories(string basePath, string pattern, ISet accumulator) - { - try - { - foreach (var dir in Directory.EnumerateDirectories(basePath, pattern, Enumeration)) - { - accumulator.Add(dir); - } - } - catch (IOException) - { - // Ignore enumeration errors - } - catch (UnauthorizedAccessException) - { - // Ignore access errors - } - } - } } diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md new file mode 100644 index 000000000..d0cb0e982 --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md @@ -0,0 +1,14 @@ +# Python Analyzer Tasks + +## Python Detection Gaps (Sprint 0405) + +| Task ID | Status | Notes | Updated (UTC) | +| --- | --- | --- | --- | +| SCAN-PY-405-001 | DONE | Wire layout-aware VFS/discovery into `PythonLanguageAnalyzer`. | 2025-12-13 | +| SCAN-PY-405-002 | BLOCKED | Preserve dist-info/egg-info evidence; emit explicit-key components where needed (incl. editable lock entries; no `@editable` PURLs). | 2025-12-13 | +| SCAN-PY-405-003 | BLOCKED | Blocked on Action 2: lock/requirements precedence + supported formats scope. | 2025-12-13 | +| SCAN-PY-405-004 | BLOCKED | Blocked on Action 3: container overlay contract (whiteouts + ordering semantics). | 2025-12-13 | +| SCAN-PY-405-005 | BLOCKED | Blocked on Action 4: vendored deps representation contract (identity/scope vs metadata-only). | 2025-12-13 | +| SCAN-PY-405-006 | BLOCKED | Blocked on Interlock 4: "used-by-entrypoint" semantics (avoid turning heuristics into truth). | 2025-12-13 | +| SCAN-PY-405-007 | BLOCKED | Blocked on Actions 2-4: fixtures for includes/editables, overlay/whiteouts, vendoring. | 2025-12-13 | +| SCAN-PY-405-008 | DONE | Docs + deterministic offline bench for Python analyzer contract. | 2025-12-13 | diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/Observations/RubyObservationBuilder.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/Observations/RubyObservationBuilder.cs index e3c3516d1..33a082560 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/Observations/RubyObservationBuilder.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/Observations/RubyObservationBuilder.cs @@ -185,11 +185,11 @@ internal static class RubyObservationBuilder string? bundledWith, RubyContainerInfo containerInfo) { - var bundlePaths = bundlerConfig.BundlePaths + var bundlePaths = bundlerConfig.BundlePathsRelative .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase) .ToImmutableArray(); - var gemfiles = bundlerConfig.Gemfiles + var gemfiles = bundlerConfig.GemfilesRelative .Select(static p => p.Replace('\\', '/')) .OrderBy(static p => p, StringComparer.OrdinalIgnoreCase) .ToImmutableArray(); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyBundlerConfig.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyBundlerConfig.cs index dc656e5f2..06f9a241c 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyBundlerConfig.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyBundlerConfig.cs @@ -2,17 +2,31 @@ namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal; internal sealed class RubyBundlerConfig { - private RubyBundlerConfig(IReadOnlyList gemfiles, IReadOnlyList bundlePaths) + private RubyBundlerConfig( + IReadOnlyList gemfiles, + IReadOnlyList bundlePaths, + IReadOnlyList gemfilesRelative, + IReadOnlyList bundlePathsRelative) { Gemfiles = gemfiles; BundlePaths = bundlePaths; + GemfilesRelative = gemfilesRelative; + BundlePathsRelative = bundlePathsRelative; } public IReadOnlyList Gemfiles { get; } public IReadOnlyList BundlePaths { get; } - public static RubyBundlerConfig Empty { get; } = new(Array.Empty(), Array.Empty()); + public IReadOnlyList GemfilesRelative { get; } + + public IReadOnlyList BundlePathsRelative { get; } + + public static RubyBundlerConfig Empty { get; } = new( + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty()); public static RubyBundlerConfig Load(string rootPath) { @@ -29,6 +43,9 @@ internal sealed class RubyBundlerConfig var gemfiles = new List(); var bundlePaths = new List(); + var gemfilesRelative = new List(); + var bundlePathsRelative = new List(); + var rootFullPath = Path.GetFullPath(rootPath); try { @@ -59,11 +76,11 @@ internal sealed class RubyBundlerConfig if (key.Equals("BUNDLE_GEMFILE", StringComparison.OrdinalIgnoreCase)) { - AddPath(gemfiles, rootPath, value); + AddPath(gemfiles, gemfilesRelative, rootFullPath, value); } else if (key.Equals("BUNDLE_PATH", StringComparison.OrdinalIgnoreCase)) { - AddPath(bundlePaths, rootPath, value); + AddPath(bundlePaths, bundlePathsRelative, rootFullPath, value); } } } @@ -77,25 +94,46 @@ internal sealed class RubyBundlerConfig } return new RubyBundlerConfig( - DistinctNormalized(gemfiles), - DistinctNormalized(bundlePaths)); + DistinctNormalizedFullPaths(gemfiles), + DistinctNormalizedFullPaths(bundlePaths), + DistinctNormalizedRelativePaths(gemfilesRelative), + DistinctNormalizedRelativePaths(bundlePathsRelative)); } - private static void AddPath(List target, string rootPath, string value) + private static void AddPath( + List absoluteTarget, + List relativeTarget, + string rootFullPath, + string value) { if (string.IsNullOrWhiteSpace(value)) { return; } - var path = Path.IsPathRooted(value) + var resolved = Path.IsPathRooted(value) ? value - : Path.Combine(rootPath, value); + : Path.Combine(rootFullPath, value); - target.Add(Path.GetFullPath(path)); + var fullPath = Path.GetFullPath(resolved); + absoluteTarget.Add(fullPath); + + var relative = Path.GetRelativePath(rootFullPath, fullPath); + if (string.IsNullOrWhiteSpace(relative) || relative == "." || Path.IsPathRooted(relative)) + { + return; + } + + var normalized = relative.Replace('\\', '/'); + if (normalized.StartsWith("../", StringComparison.Ordinal) || normalized.Equals("..", StringComparison.Ordinal)) + { + return; + } + + relativeTarget.Add(normalized); } - private static IReadOnlyList DistinctNormalized(IEnumerable values) + private static IReadOnlyList DistinctNormalizedFullPaths(IEnumerable values) { return values .Where(static value => !string.IsNullOrWhiteSpace(value)) @@ -104,4 +142,15 @@ internal sealed class RubyBundlerConfig .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) .ToArray(); } + + private static IReadOnlyList DistinctNormalizedRelativePaths(IEnumerable values) + { + return values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Replace('\\', '/')) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } } + diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyCapabilityDetector.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyCapabilityDetector.cs index 382acd67d..a29dc2cf1 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyCapabilityDetector.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyCapabilityDetector.cs @@ -47,7 +47,7 @@ internal static class RubyCapabilityDetector CreateRegex(@"\bsystem\s*\("), CreateRegex(@"\bKernel\.spawn\s*\("), CreateRegex(@"\bspawn\s*\("), - CreateRegex(@"\bOpen3\.[a-zA-Z_]+\b"), + CreateRegex(@"\bOpen3\.[a-zA-Z0-9_]+\b"), CreateRegex(@"`[^`]+`"), CreateRegex(@"%x\[[^\]]+\]"), CreateRegex(@"%x\([^)]*\)") @@ -317,4 +317,3 @@ internal static class RubyCapabilityDetector private static Regex CreateRegex(string pattern) => new(pattern, PatternOptions); } - diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyContainerScanner.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyContainerScanner.cs index c58f89271..c8531a842 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyContainerScanner.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyContainerScanner.cs @@ -184,7 +184,7 @@ internal static partial class RubyContainerScanner continue; } - ScanGemDirectory(context, fullPath, installedGems, nativeExtensions, cancellationToken); + ScanGemInstallPath(context, fullPath, installedGems, nativeExtensions, cancellationToken); } // Also scan vendor paths @@ -471,6 +471,47 @@ internal static partial class RubyContainerScanner } } + private static void ScanGemInstallPath( + LanguageAnalyzerContext context, + string rootPath, + List installedGems, + List nativeExtensions, + CancellationToken cancellationToken) + { + ScanGemDirectory(context, rootPath, installedGems, nativeExtensions, cancellationToken); + + var gemsPath = Path.Combine(rootPath, "gems"); + if (Directory.Exists(gemsPath)) + { + ScanGemDirectory(context, gemsPath, installedGems, nativeExtensions, cancellationToken); + } + + IEnumerable? versionDirectories; + try + { + versionDirectories = Directory.EnumerateDirectories(rootPath); + } + catch (IOException) + { + return; + } + catch (UnauthorizedAccessException) + { + return; + } + + foreach (var versionDirectory in versionDirectories) + { + cancellationToken.ThrowIfCancellationRequested(); + + var versionedGems = Path.Combine(versionDirectory, "gems"); + if (Directory.Exists(versionedGems)) + { + ScanGemDirectory(context, versionedGems, installedGems, nativeExtensions, cancellationToken); + } + } + } + private static void ScanVendorPaths( LanguageAnalyzerContext context, string rootPath, diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyRuntimeGraphBuilder.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyRuntimeGraphBuilder.cs index c9d1c09f2..dec884dc5 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyRuntimeGraphBuilder.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/Internal/RubyRuntimeGraphBuilder.cs @@ -46,11 +46,6 @@ internal static class RubyRuntimeGraphBuilder ArgumentNullException.ThrowIfNull(context); ArgumentNullException.ThrowIfNull(packages); - if (packages.Count == 0) - { - return RubyRuntimeGraph.Empty; - } - var usageBuilders = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var file in EnumerateRubyFiles(context.RootPath)) diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/RubyLanguageAnalyzer.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/RubyLanguageAnalyzer.cs index 4672c336d..42e5a9f9e 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/RubyLanguageAnalyzer.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/RubyLanguageAnalyzer.cs @@ -25,7 +25,7 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer var lockData = await RubyLockData.LoadAsync(context, cancellationToken).ConfigureAwait(false); var packages = RubyPackageCollector.CollectPackages(lockData, context, cancellationToken); - if (packages.Count == 0) + if (packages.Count == 0 && !LooksLikeRubyWorkspace(context.RootPath)) { return; } @@ -58,10 +58,36 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer usedByEntrypoint: runtimeUsage?.UsedByEntrypoint ?? false); } - if (packages.Count > 0) + EmitObservation(context, writer, packages, lockData, runtimeGraph, capabilities, bundlerConfig, lockData.BundledWith, containerInfo, runtimeEvidence, policyContext); + } + + private static bool LooksLikeRubyWorkspace(string rootPath) + { + if (string.IsNullOrWhiteSpace(rootPath) || !Directory.Exists(rootPath)) { - EmitObservation(context, writer, packages, lockData, runtimeGraph, capabilities, bundlerConfig, lockData.BundledWith, containerInfo, runtimeEvidence, policyContext); + return false; } + + foreach (var fileName in new[] { "Gemfile", "gems.rb", "Rakefile", "config.ru" }) + { + if (File.Exists(Path.Combine(rootPath, fileName))) + { + return true; + } + } + + var options = new EnumerationOptions + { + RecurseSubdirectories = true, + IgnoreInaccessible = true, + MaxRecursionDepth = 3, + AttributesToSkip = FileAttributes.Device | FileAttributes.ReparsePoint, + }; + + return Directory.EnumerateFiles(rootPath, "*.rb", options).Any() + || Directory.EnumerateFiles(rootPath, "*.rake", options).Any() + || Directory.EnumerateFiles(rootPath, "*.ru", options).Any() + || Directory.EnumerateFiles(rootPath, "*.thor", options).Any(); } private static async ValueTask EnsureSurfaceValidationAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken) @@ -126,6 +152,7 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer var observationMetadata = BuildObservationMetadata( packages.Count, + observationDocument.Entrypoints.Length, observationDocument.DependencyEdges.Length, observationDocument.RuntimeEdges.Length, observationDocument.Capabilities, @@ -158,6 +185,7 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer private static IEnumerable> BuildObservationMetadata( int packageCount, + int entrypointCount, int dependencyEdgeCount, int runtimeEdgeCount, RubyObservationCapabilitySummary capabilities, @@ -166,6 +194,7 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer RubyObservationRuntimeEvidence? runtimeEvidence) { yield return new KeyValuePair("ruby.observation.packages", packageCount.ToString(CultureInfo.InvariantCulture)); + yield return new KeyValuePair("ruby.observation.entrypoints", entrypointCount.ToString(CultureInfo.InvariantCulture)); yield return new KeyValuePair("ruby.observation.dependency_edges", dependencyEdgeCount.ToString(CultureInfo.InvariantCulture)); yield return new KeyValuePair("ruby.observation.runtime_edges", runtimeEdgeCount.ToString(CultureInfo.InvariantCulture)); yield return new KeyValuePair("ruby.observation.capability.exec", capabilities.UsesExec ? "true" : "false"); diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md index b061cf964..f35df929f 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/TASKS.md @@ -18,3 +18,4 @@ | `SCANNER-ANALYZERS-RUBY-28-010` | DONE (2025-11-27) | Optional runtime evidence integration with path hashing: created Internal/Runtime/ types (RubyRuntimeEvidence.cs, RubyRuntimeEvidenceCollector.cs, RubyRuntimePathHasher.cs, RubyRuntimeEvidenceIntegrator.cs). Added RubyObservationRuntimeEvidence and RubyObservationRuntimeError to observation document. Collector reads ruby-runtime.ndjson from multiple paths, parses require/load/method.call/error events, builds path hash map (SHA-256) for secure correlation. Integrator correlates package evidence, enhances runtime edges with "runtime-verified" flag, adds supplementary "runtime-only" edges without altering static precedence. Updated builder/serializer to include optional runtimeEvidence section. All 8 determinism tests pass. | | `SCANNER-ANALYZERS-RUBY-28-011` | DONE (2025-11-27) | Package analyzer plug-in, CLI, and Offline Kit docs: verified existing manifest.json (schemaVersion 1.0, capabilities: language-analyzer/ruby/rubygems/bundler, runtime-capture:optional), verified RubyAnalyzerPlugin.cs entrypoint. CLI `stella ruby inspect` and `stella ruby resolve` commands already implemented in CommandFactory.cs/CommandHandlers.cs. Updated docs/24_OFFLINE_KIT.md with comprehensive Ruby analyzer feature list covering OCI container layers, dependency edges, Ruby version detection, native extensions, web server configs, AOC-compliant observations, runtime evidence with path hashing, and CLI usage. | | `SCANNER-ANALYZERS-RUBY-28-012` | DONE (2025-11-27) | Policy signal emitter: created RubyPolicySignalEmitter.cs with signal emission for rubygems drift (declared-only, vendored, git-sourced, path-sourced counts, version mismatches), native extension flags (.so/.bundle/.dll counts, gem list), dangerous construct counts (exec/eval/serialization with risk tier), TLS posture (verify disabled, SSL context overrides, insecure HTTP), and dynamic code warnings (require/load/const_get/method_missing). Created RubyPolicyContextBuilder.cs with regex-based source scanning for dangerous patterns. Integrated into RubyLanguageAnalyzer via EmitPolicySignals. Added ScanAnalysisKeys.RubyPolicySignals key. Updated benchmark targets to 1000ms to accommodate policy scanning overhead. All 8 determinism tests pass. | +| `SCANNER-ANALYZERS-RUBY-28-013` | DOING (2025-12-13) | Fix Ruby determinism regressions (capability exec via `Open3.capture3`, container native extensions, no host paths in observation environment) and refresh golden fixtures to keep `StellaOps.Scanner.sln` green. | diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/Core/LanguageExplicitKey.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/Core/LanguageExplicitKey.cs new file mode 100644 index 000000000..f5bd5882c --- /dev/null +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/Core/LanguageExplicitKey.cs @@ -0,0 +1,26 @@ +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Scanner.Analyzers.Lang; + +public static class LanguageExplicitKey +{ + public static string Create(string analyzerId, string ecosystem, string name, string spec, string originLocator) + { + ArgumentException.ThrowIfNullOrWhiteSpace(analyzerId); + ArgumentException.ThrowIfNullOrWhiteSpace(ecosystem); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + analyzerId = analyzerId.Trim(); + ecosystem = ecosystem.Trim(); + name = name.Trim(); + spec = spec?.Trim() ?? string.Empty; + originLocator = originLocator?.Trim() ?? string.Empty; + + var canonical = string.Join('\n', ecosystem, name, spec, originLocator); + var digest = SHA256.HashData(Encoding.UTF8.GetBytes(canonical)); + var hex = Convert.ToHexString(digest).ToLowerInvariant(); + + return $"explicit::{analyzerId}::{ecosystem}::{name}::sha256:{hex}"; + } +} diff --git a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityRichGraphPublisher.cs b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityRichGraphPublisher.cs index 8a52ba7d7..22eb37b1a 100644 --- a/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityRichGraphPublisher.cs +++ b/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityRichGraphPublisher.cs @@ -1,7 +1,8 @@ using System; using System.IO; -using System.IO.Compression; -using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using StellaOps.Scanner.Cache.Abstractions; @@ -14,7 +15,7 @@ public interface IRichGraphPublisher } /// -/// Packages richgraph-v1 JSON + meta into a deterministic zip and stores it in CAS. +/// Stores richgraph-v1 JSON in CAS and emits a deterministic DSSE envelope for graph attestations. /// CAS paths follow the richgraph-v1 contract: cas://reachability/graphs/{blake3} /// public sealed class ReachabilityRichGraphPublisher : IRichGraphPublisher @@ -42,43 +43,91 @@ public sealed class ReachabilityRichGraphPublisher : IRichGraphPublisher var writeResult = await _writer.WriteAsync(graph, workRoot, analysisId, cancellationToken).ConfigureAwait(false); - var folder = Path.GetDirectoryName(writeResult.GraphPath)!; - var zipPath = Path.Combine(folder, "richgraph.zip"); - CreateDeterministicZip(folder, zipPath); - // Use BLAKE3 graph_hash as the CAS key per CONTRACT-RICHGRAPH-V1-015 var casKey = ExtractHashDigest(writeResult.GraphHash); - await using var stream = File.OpenRead(zipPath); - var casEntry = await cas.PutAsync(new FileCasPutRequest(casKey, stream, leaveOpen: false), cancellationToken).ConfigureAwait(false); + await using var graphStream = File.OpenRead(writeResult.GraphPath); + var casEntry = await cas.PutAsync(new FileCasPutRequest(casKey, graphStream, leaveOpen: false), cancellationToken).ConfigureAwait(false); // Build CAS URI per contract: cas://reachability/graphs/{blake3} var casUri = $"cas://reachability/graphs/{casKey}"; + var dsse = BuildDeterministicGraphDsse(writeResult, casUri, analysisId); + await using var dsseStream = new MemoryStream(dsse.EnvelopeJson, writable: false); + var dsseKey = $"{casKey}.dsse"; + var dsseEntry = await cas.PutAsync(new FileCasPutRequest(dsseKey, dsseStream, leaveOpen: false), cancellationToken).ConfigureAwait(false); + var dsseCasUri = $"cas://reachability/graphs/{dsseKey}"; + return new RichGraphPublishResult( writeResult.GraphHash, casEntry.RelativePath, casUri, + dsseEntry.RelativePath, + dsseCasUri, + dsse.Digest, writeResult.NodeCount, writeResult.EdgeCount); } - private static void CreateDeterministicZip(string sourceDir, string destinationZip) + private static GraphDsse BuildDeterministicGraphDsse(RichGraphWriteResult writeResult, string casUri, string analysisId) { - if (File.Exists(destinationZip)) - { - File.Delete(destinationZip); - } + var graphHash = writeResult.GraphHash; - var files = Directory.EnumerateFiles(sourceDir, "*", SearchOption.TopDirectoryOnly) - .OrderBy(f => f, StringComparer.Ordinal) - .ToList(); - - using var zip = ZipFile.Open(destinationZip, ZipArchiveMode.Create); - foreach (var file in files) + var predicate = new { - var entryName = Path.GetFileName(file); - zip.CreateEntryFromFile(file, entryName, CompressionLevel.Optimal); - } + version = "1.0", + schema = "richgraph-v1", + graphId = analysisId, + hashes = new + { + graphHash + }, + cas = new + { + location = casUri + }, + graph = new + { + nodes = new { total = writeResult.NodeCount }, + edges = new { total = writeResult.EdgeCount } + } + }; + + var payloadType = "application/vnd.stellaops.graph.predicate+json"; + var payloadBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(predicate, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = false + })); + + var signatureHex = ComputeSha256Hex(payloadBytes); + var envelope = new + { + payloadType, + payload = Base64UrlEncode(payloadBytes), + signatures = new[] + { + new { keyid = "scanner-deterministic", sig = Base64UrlEncode(Encoding.UTF8.GetBytes(signatureHex)) } + } + }; + + var envelopeJson = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(envelope, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = false + })); + + return new GraphDsse(envelopeJson, $"sha256:{signatureHex}"); + } + + private static string ComputeSha256Hex(ReadOnlySpan data) + { + Span hash = stackalloc byte[32]; + SHA256.HashData(data, hash); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string Base64UrlEncode(ReadOnlySpan data) + { + var base64 = Convert.ToBase64String(data); + return base64.Replace("+", "-").Replace("/", "_").TrimEnd('='); } /// @@ -95,5 +144,10 @@ public sealed record RichGraphPublishResult( string GraphHash, string RelativePath, string CasUri, + string DsseRelativePath, + string DsseCasUri, + string DsseDigest, int NodeCount, int EdgeCount); + +internal sealed record GraphDsse(byte[] EnvelopeJson, string Digest); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Bun/BunLanguageAnalyzerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Bun/BunLanguageAnalyzerTests.cs index 33deae664..99dcddfb4 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Bun/BunLanguageAnalyzerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Bun/BunLanguageAnalyzerTests.cs @@ -196,6 +196,25 @@ public sealed class BunLanguageAnalyzerTests cancellationToken); } + [Fact] + public async Task PatchedMultiVersionIsParsedAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "bun", "patched-multi-version"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] + { + new BunLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + [Fact] public async Task DeepDependencyTreeIsParsedAsync() { @@ -252,4 +271,80 @@ public sealed class BunLanguageAnalyzerTests analyzers, cancellationToken); } + + [Fact] + public async Task ContainerLayersAreDiscoveredAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "bun", "container-layers"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] + { + new BunLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + + [Fact] + public async Task BunfigOnlyEmitsDeclaredOnlyAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "bun", "bunfig-only"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] + { + new BunLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + + [Fact] + public async Task LockfileDevClassificationIsDeterministicAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "bun", "lockfile-dev-classification"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] + { + new BunLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + + [Fact] + public async Task NonConcreteVersionsUseExplicitKeyAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "bun", "non-concrete-versions"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] + { + new BunLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/bunfig.toml b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/bunfig.toml new file mode 100644 index 000000000..b6199a0a1 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/bunfig.toml @@ -0,0 +1,2 @@ +[install] +cache = false diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/expected.json new file mode 100644 index 000000000..55399acd7 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/expected.json @@ -0,0 +1,74 @@ +[ + { + "analyzerId": "bun", + "componentKey": "explicit::bun::npm::left-pad::sha256:8ad9c18ee1a619ce3a224346fe984c4ced211ac443ebf7d709a93f1343ef8ba2", + "name": "left-pad", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "prod", + "declared.source": "package.json", + "declared.sourceType": "range", + "declared.versionSpec": "^1.3.0", + "declaredOnly": "true", + "packageManager": "bun" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "package.json", + "sha256": "465919e1195aa0b066f473c55341df77abff6a6b7d62e25d63ccfb7c13e3287b" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "explicit::bun::npm::local-file::sha256:61b6ef7b8e24fe3a1e1080296c61f2ca4ad8839f453e24cb8adf874678521caa", + "name": "local-file", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "prod", + "declared.source": "package.json", + "declared.sourceType": "file", + "declared.versionSpec": "file:../local-file", + "declaredOnly": "true", + "packageManager": "bun" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "package.json", + "sha256": "465919e1195aa0b066f473c55341df77abff6a6b7d62e25d63ccfb7c13e3287b" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "explicit::bun::npm::typescript::sha256:5a0a88f051ea20b8875334dadc5bce3c0861d146b151ab7bab95654541b7a168", + "name": "typescript", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#devDependencies", + "declared.scope": "dev", + "declared.source": "package.json", + "declared.sourceType": "range", + "declared.versionSpec": "~5.3.0", + "declaredOnly": "true", + "packageManager": "bun" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "package.json", + "sha256": "465919e1195aa0b066f473c55341df77abff6a6b7d62e25d63ccfb7c13e3287b" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/package.json new file mode 100644 index 000000000..dbd5326d3 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/bunfig-only/package.json @@ -0,0 +1,11 @@ +{ + "name": "bunfig-only-fixture", + "private": true, + "dependencies": { + "left-pad": "^1.3.0", + "local-file": "file:../local-file" + }, + "devDependencies": { + "typescript": "~5.3.0" + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/bun.lock b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/bun.lock new file mode 100644 index 000000000..9e5632372 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/bun.lock @@ -0,0 +1,6 @@ +{ + "lockfileVersion": 1, + "packages": { + "ms@2.1.3": ["https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="] + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/package.json new file mode 100644 index 000000000..f73016612 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/.layers/layer0/app/package.json @@ -0,0 +1,7 @@ +{ + "name": "bun-container-layers-fixture", + "version": "1.0.0", + "dependencies": { + "ms": "^2.1.3" + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/expected.json new file mode 100644 index 000000000..2ca679174 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/container-layers/expected.json @@ -0,0 +1,34 @@ +[ + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/ms@2.1.3", + "purl": "pkg:npm/ms@2.1.3", + "name": "ms", + "version": "2.1.3", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "packageManager": "bun", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "source": "bun.lock" + }, + "evidence": [ + { + "kind": "metadata", + "source": "integrity", + "locator": ".layers/layer0/app/bun.lock:packages[ms@2.1.3]", + "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "sha256": "4a384b14aba7740bd500cdf0da7329a41a2940662e9b1fcab1fbc71c6c8389e7" + }, + { + "kind": "metadata", + "source": "resolved", + "locator": ".layers/layer0/app/bun.lock:packages[ms@2.1.3]", + "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "sha256": "4a384b14aba7740bd500cdf0da7329a41a2940662e9b1fcab1fbc71c6c8389e7" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/custom-registry/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/custom-registry/expected.json index 33a574910..37ad1eadd 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/custom-registry/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/custom-registry/expected.json @@ -22,20 +22,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/@company/internal-pkg/package.json" + "locator": "node_modules/@company/internal-pkg/package.json", + "sha256": "f5311f43a95bd76e1912dbd7d0a5b3611baa9e82bcf72d5dc7f34c5f71f0ddf4" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-customhash123==" + "locator": "bun.lock:packages[@company/internal-pkg@1.0.0]", + "value": "sha512-customhash123==", + "sha256": "eb3bacf736d4a1b3cf9e02357afc1add9f20323916ce62cf8748c9ad9a80f195" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://npm.company.com/@company/internal-pkg/-/internal-pkg-1.0.0.tgz" + "locator": "bun.lock:packages[@company/internal-pkg@1.0.0]", + "value": "https://npm.company.com/@company/internal-pkg/-/internal-pkg-1.0.0.tgz", + "sha256": "eb3bacf736d4a1b3cf9e02357afc1add9f20323916ce62cf8748c9ad9a80f195" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/deep-tree/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/deep-tree/expected.json index 70f73270d..d6c5a293c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/deep-tree/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/deep-tree/expected.json @@ -19,19 +19,22 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/debug/package.json" + "locator": "node_modules/debug/package.json", + "sha256": "2258b5b4d7e5ed711aeef1a86d5d9e5abf2a04410e05bd89ea806e423417e493" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX\u002B7G/vCNNhehwxfkQ==" + "locator": "bun.lock:packages[debug@4.3.4]", + "value": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX\u002B7G/vCNNhehwxfkQ==", + "sha256": "33d4886c0591242ffb78b5e739c5248c81559312586d59d543d48387e4bb6a2b" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" + "locator": "bun.lock:packages[debug@4.3.4]", + "value": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "sha256": "33d4886c0591242ffb78b5e739c5248c81559312586d59d543d48387e4bb6a2b" } ] }, @@ -54,20 +57,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/ms/package.json" + "locator": "node_modules/ms/package.json", + "sha256": "ae11c4ce44027a95893e8c890aed0c582f04e8cf1b8022931eddcb613cd9d3f7" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "locator": "bun.lock:packages[ms@2.1.3]", + "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "sha256": "33d4886c0591242ffb78b5e739c5248c81559312586d59d543d48387e4bb6a2b" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + "locator": "bun.lock:packages[ms@2.1.3]", + "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "sha256": "33d4886c0591242ffb78b5e739c5248c81559312586d59d543d48387e4bb6a2b" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/git-dependencies/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/git-dependencies/expected.json index 38e9128d3..4242d7f09 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/git-dependencies/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/git-dependencies/expected.json @@ -21,14 +21,16 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/my-git-pkg/package.json" + "locator": "node_modules/my-git-pkg/package.json", + "sha256": "45687abed9d301c361987ca877da135e830c80dc3ce37f9ea1c74c7df96b8bf2" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "git\u002Bhttps://github.com/user/my-git-pkg.git#abc123def456" + "locator": "bun.lock:packages[my-git-pkg@1.0.0]", + "value": "git\u002Bhttps://github.com/user/my-git-pkg.git#abc123def456", + "sha256": "819a7efc185bd1314d21aa7fdc0e5b2134a0c9b758ecd9daa62cb6cba2feddd0" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/isolated/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/isolated/expected.json index 6a3d01091..0a7a228a5 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/isolated/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/isolated/expected.json @@ -18,19 +18,22 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/.bun/is-number@6.0.0/package.json" + "locator": "node_modules/.bun/is-number@6.0.0/package.json", + "sha256": "0324c895ec4aa4049c77371f08e937eed97a58e442595a8834ba21afd8e100b3" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-Wu1VZAVuL1snqOnHLxJ0l2p3pjlzLnMcJ8gJhaTZVfP7VFKN7fSJ8X/gR0qFCLwfFJ0Rqd3IxfS\u002BTY/Lc1Q7Pw==" + "locator": "bun.lock:packages[is-number@6.0.0]", + "value": "sha512-Wu1VZAVuL1snqOnHLxJ0l2p3pjlzLnMcJ8gJhaTZVfP7VFKN7fSJ8X/gR0qFCLwfFJ0Rqd3IxfS\u002BTY/Lc1Q7Pw==", + "sha256": "746b6c809e50ee2d7bdb27a0ee43046d48fa5f21d7597bbadd3bd44269798812" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/is-number/-/is-number-6.0.0.tgz" + "locator": "bun.lock:packages[is-number@6.0.0]", + "value": "https://registry.npmjs.org/is-number/-/is-number-6.0.0.tgz", + "sha256": "746b6c809e50ee2d7bdb27a0ee43046d48fa5f21d7597bbadd3bd44269798812" } ] }, @@ -54,20 +57,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/.bun/is-odd@3.0.1/package.json" + "locator": "node_modules/.bun/is-odd@3.0.1/package.json", + "sha256": "beb18158821ecb86f3bb2a6be3ef817c0b8dcdc3e05a53e0b9a1c62d74a595ac" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-CQpnWPrDwmP1\u002BSMHXvTXAoSEu2mCPgMU0VKt1WcA7D8VXCo4HfVNlUbD1k8Tg0BVDX/LhyRaZqKqiS4vI6tTHg==" + "locator": "bun.lock:packages[is-odd@3.0.1]", + "value": "sha512-CQpnWPrDwmP1\u002BSMHXvTXAoSEu2mCPgMU0VKt1WcA7D8VXCo4HfVNlUbD1k8Tg0BVDX/LhyRaZqKqiS4vI6tTHg==", + "sha256": "746b6c809e50ee2d7bdb27a0ee43046d48fa5f21d7597bbadd3bd44269798812" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/is-odd/-/is-odd-3.0.1.tgz" + "locator": "bun.lock:packages[is-odd@3.0.1]", + "value": "https://registry.npmjs.org/is-odd/-/is-odd-3.0.1.tgz", + "sha256": "746b6c809e50ee2d7bdb27a0ee43046d48fa5f21d7597bbadd3bd44269798812" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/jsonc-lockfile/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/jsonc-lockfile/expected.json index fa6cfbf46..fc5657462 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/jsonc-lockfile/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/jsonc-lockfile/expected.json @@ -19,20 +19,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/lodash/package.json" + "locator": "node_modules/lodash/package.json", + "sha256": "82145cd4bdc9a690c14843b405179c60aeda1a958029f6ae62776c1b26e42169" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==", + "sha256": "7b34fdbdf0cb3e0d07e25f7d7f452491dcfad421138449217a1c20b2f66a6475" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "sha256": "7b34fdbdf0cb3e0d07e25f7d7f452491dcfad421138449217a1c20b2f66a6475" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/bun.lock b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/bun.lock new file mode 100644 index 000000000..b45af939e --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/bun.lock @@ -0,0 +1,10 @@ +{ + "lockfileVersion": 1, + "packages": { + "prod-pkg@1.0.0": ["https://registry.npmjs.org/prod-pkg/-/prod-pkg-1.0.0.tgz", null, {"shared": "^1.0.0"}], + "dev-pkg@1.0.0": ["https://registry.npmjs.org/dev-pkg/-/dev-pkg-1.0.0.tgz", null, {"dev-only": "^1.0.0"}], + "shared@1.0.0": ["https://registry.npmjs.org/shared/-/shared-1.0.0.tgz", null], + "dev-only@1.0.0": ["https://registry.npmjs.org/dev-only/-/dev-only-1.0.0.tgz", null] + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/expected.json new file mode 100644 index 000000000..8b52016ec --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/expected.json @@ -0,0 +1,98 @@ +[ + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/dev-only@1.0.0", + "purl": "pkg:npm/dev-only@1.0.0", + "name": "dev-only", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "dev": "true", + "packageManager": "bun", + "resolved": "https://registry.npmjs.org/dev-only/-/dev-only-1.0.0.tgz", + "source": "bun.lock" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[dev-only@1.0.0]", + "value": "https://registry.npmjs.org/dev-only/-/dev-only-1.0.0.tgz", + "sha256": "4d40cc185e492e4544a6dc3b17cdfd77096e4d4260569a243eb694befbada6ac" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/dev-pkg@1.0.0", + "purl": "pkg:npm/dev-pkg@1.0.0", + "name": "dev-pkg", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "dev": "true", + "direct": "true", + "packageManager": "bun", + "resolved": "https://registry.npmjs.org/dev-pkg/-/dev-pkg-1.0.0.tgz", + "source": "bun.lock" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[dev-pkg@1.0.0]", + "value": "https://registry.npmjs.org/dev-pkg/-/dev-pkg-1.0.0.tgz", + "sha256": "4d40cc185e492e4544a6dc3b17cdfd77096e4d4260569a243eb694befbada6ac" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/prod-pkg@1.0.0", + "purl": "pkg:npm/prod-pkg@1.0.0", + "name": "prod-pkg", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "packageManager": "bun", + "resolved": "https://registry.npmjs.org/prod-pkg/-/prod-pkg-1.0.0.tgz", + "source": "bun.lock" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[prod-pkg@1.0.0]", + "value": "https://registry.npmjs.org/prod-pkg/-/prod-pkg-1.0.0.tgz", + "sha256": "4d40cc185e492e4544a6dc3b17cdfd77096e4d4260569a243eb694befbada6ac" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/shared@1.0.0", + "purl": "pkg:npm/shared@1.0.0", + "name": "shared", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "packageManager": "bun", + "resolved": "https://registry.npmjs.org/shared/-/shared-1.0.0.tgz", + "source": "bun.lock" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[shared@1.0.0]", + "value": "https://registry.npmjs.org/shared/-/shared-1.0.0.tgz", + "sha256": "4d40cc185e492e4544a6dc3b17cdfd77096e4d4260569a243eb694befbada6ac" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/package.json new file mode 100644 index 000000000..1e4e21bc0 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-dev-classification/package.json @@ -0,0 +1,11 @@ +{ + "name": "bun-lockfile-dev-classification-fixture", + "version": "1.0.0", + "dependencies": { + "prod-pkg": "^1.0.0" + }, + "devDependencies": { + "dev-pkg": "^1.0.0" + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-only/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-only/expected.json index 292a5add6..ec72cb9a5 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-only/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/lockfile-only/expected.json @@ -18,15 +18,17 @@ { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "locator": "bun.lock:packages[ms@2.1.3]", + "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "sha256": "4a384b14aba7740bd500cdf0da7329a41a2940662e9b1fcab1fbc71c6c8389e7" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + "locator": "bun.lock:packages[ms@2.1.3]", + "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "sha256": "4a384b14aba7740bd500cdf0da7329a41a2940662e9b1fcab1fbc71c6c8389e7" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/expected.json index df7c0c6f4..73d6bd3fb 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/expected.json @@ -19,19 +19,22 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/lodash/package.json" + "locator": "node_modules/lodash/package.json", + "sha256": "82145cd4bdc9a690c14843b405179c60aeda1a958029f6ae62776c1b26e42169" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==", + "sha256": "8a0d37c3761b81514ee397c3836ccff48167ce6aa1afdfd484ca7679e586df4a" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "sha256": "8a0d37c3761b81514ee397c3836ccff48167ce6aa1afdfd484ca7679e586df4a" } ] }, @@ -55,20 +58,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/ms/package.json" + "locator": "node_modules/ms/package.json", + "sha256": "ae11c4ce44027a95893e8c890aed0c582f04e8cf1b8022931eddcb613cd9d3f7" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + "locator": "bun.lock:packages[ms@2.1.3]", + "value": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "sha256": "8a0d37c3761b81514ee397c3836ccff48167ce6aa1afdfd484ca7679e586df4a" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + "locator": "bun.lock:packages[ms@2.1.3]", + "value": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "sha256": "8a0d37c3761b81514ee397c3836ccff48167ce6aa1afdfd484ca7679e586df4a" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/package.json new file mode 100644 index 000000000..f2aaef21a --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/multi-workspace/package.json @@ -0,0 +1,11 @@ +{ + "name": "bun-multi-workspace-fixture", + "version": "1.0.0", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "lodash": "^4.17.21", + "ms": "^2.1.3" + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/bun.lock b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/bun.lock new file mode 100644 index 000000000..2986378e8 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/bun.lock @@ -0,0 +1,21 @@ +{ + "lockfileVersion": 1, + "packages": { + "file-pkg@file:../file-pkg.tgz": [ + "file:../file-pkg.tgz", + null, + {} + ], + "link-pkg@link:../link-pkg": [ + "link:../link-pkg", + null, + {} + ], + "local-pkg@workspace:*": [ + "workspace:packages/local-pkg", + null, + {} + ] + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/expected.json new file mode 100644 index 000000000..6e55648d8 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/expected.json @@ -0,0 +1,80 @@ +[ + { + "analyzerId": "bun", + "componentKey": "explicit::bun::npm::file-pkg::sha256:c541f5764a7e2fdea9fc5789b13e404f8e15ffc8db0110a81346552c607c89ff", + "name": "file-pkg", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "nonConcreteVersion": "true", + "packageManager": "bun", + "resolved": "file:../file-pkg.tgz", + "source": "bun.lock", + "sourceType": "file", + "specifier": "file:../file-pkg.tgz", + "versionSpec": "file:../file-pkg.tgz" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[file-pkg@file:../file-pkg.tgz]", + "value": "file:../file-pkg.tgz", + "sha256": "d7ae02476b6737ea3056226ea69e36bacb664feacd7a5223bc66ea287757656b" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "explicit::bun::npm::link-pkg::sha256:ebb0c3119ab319e05c02e2448f1d6b4a23dc69076bf8dbfbe95657a3405d1b11", + "name": "link-pkg", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "nonConcreteVersion": "true", + "packageManager": "bun", + "resolved": "link:../link-pkg", + "source": "bun.lock", + "sourceType": "link", + "specifier": "link:../link-pkg", + "versionSpec": "link:../link-pkg" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[link-pkg@link:../link-pkg]", + "value": "link:../link-pkg", + "sha256": "d7ae02476b6737ea3056226ea69e36bacb664feacd7a5223bc66ea287757656b" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "explicit::bun::npm::local-pkg::sha256:cafb6902358bb6a2503a67d71abe50446cdcb3c8359dfb6f3ab00ee1672a5c07", + "name": "local-pkg", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "nonConcreteVersion": "true", + "packageManager": "bun", + "resolved": "workspace:packages/local-pkg", + "source": "bun.lock", + "sourceType": "workspace", + "specifier": "workspace:packages/local-pkg", + "versionSpec": "workspace:*" + }, + "evidence": [ + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[local-pkg@workspace:*]", + "value": "workspace:packages/local-pkg", + "sha256": "d7ae02476b6737ea3056226ea69e36bacb664feacd7a5223bc66ea287757656b" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/package.json new file mode 100644 index 000000000..4e7d27c67 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/non-concrete-versions/package.json @@ -0,0 +1,10 @@ +{ + "name": "non-concrete-versions", + "version": "1.0.0", + "dependencies": { + "file-pkg": "file:../file-pkg.tgz", + "link-pkg": "link:../link-pkg", + "local-pkg": "workspace:*" + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/bun.lock b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/bun.lock new file mode 100644 index 000000000..47f330e11 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/bun.lock @@ -0,0 +1,8 @@ +{ + "lockfileVersion": 1, + "packages": { + "lodash@4.17.21": ["https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "sha512-lodash-421"], + "lodash@4.17.20": ["https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", "sha512-lodash-420"] + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/expected.json new file mode 100644 index 000000000..9acdac70e --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/expected.json @@ -0,0 +1,86 @@ +[ + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/lodash@4.17.20", + "purl": "pkg:npm/lodash@4.17.20", + "name": "lodash", + "version": "4.17.20", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "integrity": "sha512-lodash-420", + "packageManager": "bun", + "patchFile": "patches/lodash@4.17.20.patch", + "patched": "true", + "path": "node_modules/a/node_modules/lodash", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "scopeUnknown": "true", + "source": "node_modules" + }, + "evidence": [ + { + "kind": "file", + "source": "node_modules", + "locator": "node_modules/a/node_modules/lodash/package.json", + "sha256": "a883443850ed2188979ee56e2cf8200fa34935a65aae606d85d1aaa60d8ff32e" + }, + { + "kind": "metadata", + "source": "integrity", + "locator": "bun.lock:packages[lodash@4.17.20]", + "value": "sha512-lodash-420", + "sha256": "e83cd6aa810c1a8af47d6ae0eb621a8a5dc13b23ec08925ad9b5ff4d035cfc7c" + }, + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[lodash@4.17.20]", + "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "sha256": "e83cd6aa810c1a8af47d6ae0eb621a8a5dc13b23ec08925ad9b5ff4d035cfc7c" + } + ] + }, + { + "analyzerId": "bun", + "componentKey": "purl::pkg:npm/lodash@4.17.21", + "purl": "pkg:npm/lodash@4.17.21", + "name": "lodash", + "version": "4.17.21", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "direct": "true", + "integrity": "sha512-lodash-421", + "packageManager": "bun", + "patchFile": "patches/lodash@4.17.21.patch", + "patched": "true", + "path": "node_modules/lodash", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "scopeUnknown": "true", + "source": "node_modules" + }, + "evidence": [ + { + "kind": "file", + "source": "node_modules", + "locator": "node_modules/lodash/package.json", + "sha256": "1bb77ea984b96ef61781adcc6299a2a1c5f9e42dcf594264cdbb96aa06f5c813" + }, + { + "kind": "metadata", + "source": "integrity", + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "sha512-lodash-421", + "sha256": "e83cd6aa810c1a8af47d6ae0eb621a8a5dc13b23ec08925ad9b5ff4d035cfc7c" + }, + { + "kind": "metadata", + "source": "resolved", + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "sha256": "e83cd6aa810c1a8af47d6ae0eb621a8a5dc13b23ec08925ad9b5ff4d035cfc7c" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/package.json new file mode 100644 index 000000000..7be805f2f --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/package.json @@ -0,0 +1,12 @@ +{ + "name": "patched-multi-version-fixture", + "version": "1.0.0", + "dependencies": { + "lodash": "^4.17.21" + }, + "patchedDependencies": { + "lodash@4.17.21": "patches/lodash@4.17.21.patch", + "lodash@4.17.20": "patches/lodash@4.17.20.patch" + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.20.patch b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.20.patch new file mode 100644 index 000000000..0d71bb1e9 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.20.patch @@ -0,0 +1,8 @@ +diff --git a/index.js b/index.js +index 0000000..2222222 100644 +--- a/index.js ++++ b/index.js +@@ +-// placeholder ++// patched + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.21.patch b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.21.patch new file mode 100644 index 000000000..d79bb71ad --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-multi-version/patches/lodash@4.17.21.patch @@ -0,0 +1,8 @@ +diff --git a/index.js b/index.js +index 0000000..1111111 100644 +--- a/index.js ++++ b/index.js +@@ +-// placeholder ++// patched + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-packages/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-packages/expected.json index 81ee4d585..db99abd6e 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-packages/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/patched-packages/expected.json @@ -21,20 +21,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/lodash/package.json" + "locator": "node_modules/lodash/package.json", + "sha256": "82145cd4bdc9a690c14843b405179c60aeda1a958029f6ae62776c1b26e42169" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==", + "sha256": "61ff5c565c08f6564bd16153c10feba4a171986510aaf40f84fe710eabd180c2" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "sha256": "61ff5c565c08f6564bd16153c10feba4a171986510aaf40f84fe710eabd180c2" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/scoped-packages/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/scoped-packages/expected.json index 2b284bea7..c67530d22 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/scoped-packages/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/scoped-packages/expected.json @@ -19,19 +19,22 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/@babel/core/package.json" + "locator": "node_modules/@babel/core/package.json", + "sha256": "c4d995bed6c0ec71ccf6ecb74ee8f20b1431798bd93e54182afcb6870b6cfa23" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR\u002BK9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==" + "locator": "bun.lock:packages[@babel/core@7.24.0]", + "value": "sha512-fQfkg0Gjkza3nf0c7/w6Xf34BW4YvzNfACRLmmb7XRLa6XHdR\u002BK9AlJlxneFfWYf6uhOzuzZVTjF/8KfndZANw==", + "sha256": "6ffde82e85e550d36bdb577210cd80c56cbd36c02dbfb4d8ec6ada27643bcd2d" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz" + "locator": "bun.lock:packages[@babel/core@7.24.0]", + "value": "https://registry.npmjs.org/@babel/core/-/core-7.24.0.tgz", + "sha256": "6ffde82e85e550d36bdb577210cd80c56cbd36c02dbfb4d8ec6ada27643bcd2d" } ] }, @@ -55,20 +58,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/@types/node/package.json" + "locator": "node_modules/@types/node/package.json", + "sha256": "db7446931abf3479f92734485e30ee7631923d056bcfa5b210159008524f40e2" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-o9bjXmDNcF7GbM4CNQpmi\u002BTutCgap/K3w1JyKgxXjVJa7b8XWCF/wPH2E/0Vz9e\u002BV1B3eXX0WCw\u002BINcAobvUag==" + "locator": "bun.lock:packages[@types/node@20.11.0]", + "value": "sha512-o9bjXmDNcF7GbM4CNQpmi\u002BTutCgap/K3w1JyKgxXjVJa7b8XWCF/wPH2E/0Vz9e\u002BV1B3eXX0WCw\u002BINcAobvUag==", + "sha256": "6ffde82e85e550d36bdb577210cd80c56cbd36c02dbfb4d8ec6ada27643bcd2d" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz" + "locator": "bun.lock:packages[@types/node@20.11.0]", + "value": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz", + "sha256": "6ffde82e85e550d36bdb577210cd80c56cbd36c02dbfb4d8ec6ada27643bcd2d" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/standard/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/standard/expected.json index fa6cfbf46..ba53bb909 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/standard/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/standard/expected.json @@ -19,20 +19,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/lodash/package.json" + "locator": "node_modules/lodash/package.json", + "sha256": "bfe21067561ba47f62c290400e6208b95ac875f0c41e00c4dddce889e8a8ad4e" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vz1kAmtILi\u002B8fm9nJMg7b0GN8sMEJz2mxG/S7mNxhWQ7\u002BD9bF8Q==", + "sha256": "61ff5c565c08f6564bd16153c10feba4a171986510aaf40f84fe710eabd180c2" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + "locator": "bun.lock:packages[lodash@4.17.21]", + "value": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "sha256": "61ff5c565c08f6564bd16153c10feba4a171986510aaf40f84fe710eabd180c2" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/symlinks/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/symlinks/expected.json index 341661730..b6aad5567 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/symlinks/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/symlinks/expected.json @@ -19,20 +19,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/safe-pkg/package.json" + "locator": "node_modules/safe-pkg/package.json", + "sha256": "1ade6129984f59a954ec2c175075e74cb2759ba97b9b04acf76537262b0f35af" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-abc123" + "locator": "bun.lock:packages[safe-pkg@1.0.0]", + "value": "sha512-abc123", + "sha256": "54dd0b2c2f30e59b29970d34350d083b295789e056e849361da5be932d1ef747" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/safe-pkg/-/safe-pkg-1.0.0.tgz" + "locator": "bun.lock:packages[safe-pkg@1.0.0]", + "value": "https://registry.npmjs.org/safe-pkg/-/safe-pkg-1.0.0.tgz", + "sha256": "54dd0b2c2f30e59b29970d34350d083b295789e056e849361da5be932d1ef747" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/workspaces/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/workspaces/expected.json index 1a0dec4f0..0543a5885 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/workspaces/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Fixtures/lang/bun/workspaces/expected.json @@ -19,20 +19,23 @@ { "kind": "file", "source": "node_modules", - "locator": "node_modules/chalk/package.json" + "locator": "node_modules/chalk/package.json", + "sha256": "7d6ff4f365c8d42bae13a48bb4bc84e4cef4e7a7bd7b211e0662ef62fb675736" }, { "kind": "metadata", "source": "integrity", - "locator": "bun.lock", - "value": "sha512-dLitG79d\u002BGV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos\u002Buw7WmWF4wUwBd9jxjocFC2w==" + "locator": "bun.lock:packages[chalk@5.3.0]", + "value": "sha512-dLitG79d\u002BGV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos\u002Buw7WmWF4wUwBd9jxjocFC2w==", + "sha256": "8706c5aecdc68ae4f06c6a2f1bfa9e431e473a961c2f32063911febaba0c65cc" }, { "kind": "metadata", "source": "resolved", - "locator": "bun.lock", - "value": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz" + "locator": "bun.lock:packages[chalk@5.3.0]", + "value": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "sha256": "8706c5aecdc68ae4f06c6a2f1bfa9e431e473a961c2f32063911febaba0c65cc" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockParserTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockParserTests.cs index 15f7cd1b1..16619f33d 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockParserTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockParserTests.cs @@ -322,7 +322,31 @@ public sealed class BunLockParserTests Assert.Single(result.AllEntries); var entry = result.AllEntries[0]; Assert.Single(entry.Dependencies); - Assert.Contains("ms", entry.Dependencies); + Assert.Equal("ms", entry.Dependencies[0].Name); + Assert.Equal("^2.1.3", entry.Dependencies[0].Specifier); + Assert.False(entry.Dependencies[0].IsOptionalPeer); + } + + [Fact] + public void Parse_ArrayFormat_ExtractsOptionalPeerDependencies() + { + var content = """ + { + "lockfileVersion": 1, + "packages": { + "pkg@1.0.0": ["https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", null, {}, {"react": "^18.0.0"}] + } + } + """; + + var result = BunLockParser.Parse(content); + + Assert.Single(result.AllEntries); + var entry = result.AllEntries[0]; + Assert.Single(entry.Dependencies); + Assert.Equal("react", entry.Dependencies[0].Name); + Assert.Equal("^18.0.0", entry.Dependencies[0].Specifier); + Assert.True(entry.Dependencies[0].IsOptionalPeer); } [Fact] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockScopeClassifierTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockScopeClassifierTests.cs new file mode 100644 index 000000000..867f82858 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunLockScopeClassifierTests.cs @@ -0,0 +1,30 @@ +using StellaOps.Scanner.Analyzers.Lang.Bun.Internal; +using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +namespace StellaOps.Scanner.Analyzers.Lang.Bun.Tests.Parsers; + +public sealed class BunLockScopeClassifierTests +{ + [Fact] + public void IncludeDevFalse_FiltersDevOnlyPackages() + { + var fixturePath = TestPaths.ResolveFixture("lang", "bun", "lockfile-dev-classification"); + var lockPath = Path.Combine(fixturePath, "bun.lock"); + + var lockData = BunLockParser.Parse(File.ReadAllText(lockPath)); + var declared = BunDeclaredDependencyCollector.Collect(fixturePath); + var classified = BunLockScopeClassifier.Classify(lockData, declared); + + Assert.True(classified.FindEntry("dev-pkg", "1.0.0")?.IsDev ?? false); + Assert.True(classified.FindEntry("dev-only", "1.0.0")?.IsDev ?? false); + Assert.False(classified.FindEntry("prod-pkg", "1.0.0")?.IsDev ?? true); + Assert.False(classified.FindEntry("shared", "1.0.0")?.IsDev ?? true); + + var filtered = BunLockInventory.ExtractPackages(classified, includeDev: false); + + Assert.DoesNotContain(filtered, package => package.Name == "dev-pkg"); + Assert.DoesNotContain(filtered, package => package.Name == "dev-only"); + Assert.Contains(filtered, package => package.Name == "prod-pkg"); + Assert.Contains(filtered, package => package.Name == "shared"); + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunPackageTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunPackageTests.cs index 9e66425ba..2823cae44 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunPackageTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunPackageTests.cs @@ -222,7 +222,7 @@ public sealed class BunPackageTests var resolvedEvidence = evidence.FirstOrDefault(e => e.Source == "resolved"); Assert.NotNull(resolvedEvidence); Assert.Equal(LanguageEvidenceKind.Metadata, resolvedEvidence.Kind); - Assert.Equal("bun.lock", resolvedEvidence.Locator); + Assert.Equal("bun.lock:packages[lodash@4.17.21]", resolvedEvidence.Locator); Assert.Equal("https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", resolvedEvidence.Value); // Integrity evidence @@ -265,7 +265,7 @@ public sealed class BunPackageTests IsOptional = false, IsPeer = false, SourceType = "npm", - Dependencies = new List { "debug" } + Dependencies = new List { new("debug", "^4.3.4", IsOptionalPeer: false) } }; var package = BunPackage.FromLockEntry(lockEntry, "bun.lock"); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunWorkspaceHelperTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunWorkspaceHelperTests.cs index 1d44f900f..7faed3eea 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunWorkspaceHelperTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/Parsers/BunWorkspaceHelperTests.cs @@ -192,8 +192,8 @@ public sealed class BunWorkspaceHelperTests : IDisposable var result = BunWorkspaceHelper.ParseWorkspaceInfo(_tempDir); Assert.Single(result.PatchedDependencies); - Assert.True(result.PatchedDependencies.ContainsKey("lodash")); - Assert.Equal("patches/lodash@4.17.21.patch", result.PatchedDependencies["lodash"]); + Assert.True(result.PatchedDependencies.ContainsKey("lodash@4.17.21")); + Assert.Equal("patches/lodash@4.17.21.patch", result.PatchedDependencies["lodash@4.17.21"]); } [Fact] @@ -215,8 +215,10 @@ public sealed class BunWorkspaceHelperTests : IDisposable var result = BunWorkspaceHelper.ParseWorkspaceInfo(_tempDir); Assert.Equal(2, result.PatchedDependencies.Count); - Assert.True(result.PatchedDependencies.ContainsKey("lodash")); - Assert.True(result.PatchedDependencies.ContainsKey("@babel+core")); + Assert.True(result.PatchedDependencies.ContainsKey("lodash@4.17.21")); + Assert.Equal("patches/lodash@4.17.21.patch", result.PatchedDependencies["lodash@4.17.21"]); + Assert.True(result.PatchedDependencies.ContainsKey("@babel/core@7.24.0")); + Assert.Equal("patches/@babel+core@7.24.0.patch", result.PatchedDependencies["@babel/core@7.24.0"]); } [Fact] @@ -237,7 +239,8 @@ public sealed class BunWorkspaceHelperTests : IDisposable var result = BunWorkspaceHelper.ParseWorkspaceInfo(_tempDir); Assert.Single(result.PatchedDependencies); - Assert.True(result.PatchedDependencies.ContainsKey("ms")); + Assert.True(result.PatchedDependencies.ContainsKey("ms@2.1.3")); + Assert.Equal(".patches/ms@2.1.3.patch", result.PatchedDependencies["ms@2.1.3"]); } [Fact] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests.csproj index 42f3f69f2..9caac0bac 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests/StellaOps.Scanner.Analyzers.Lang.Bun.Tests.csproj @@ -17,12 +17,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimePathHasherTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimePathHasherTests.cs index 4347db485..9a397428e 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimePathHasherTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimePathHasherTests.cs @@ -17,7 +17,7 @@ public sealed class DenoRuntimePathHasherTests var identity = DenoRuntimePathHasher.Create(root, absolute); Assert.Equal("subdir/main.ts", identity.Normalized); - Assert.Equal("2d0ef79c25b433a216f41853e89d8e1e1e1ef0b0e77d12b37a7f4f7c2a25f635", identity.PathSha256); + Assert.Equal("c3b59fd8169cee9cc111b4737e733f8c0227403717e04f37cba870c49c7ff2c3", identity.PathSha256); } finally { @@ -33,7 +33,7 @@ public sealed class DenoRuntimePathHasherTests { var identity = DenoRuntimePathHasher.Create(root, root); Assert.Equal(".", identity.Normalized); - Assert.Equal("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", identity.PathSha256); + Assert.Equal("cdb4ee2aea69cc6a83331bbe96dc2caa9a299d21329efb0336fc02a82e1839a8", identity.PathSha256); } finally { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceProbeTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceProbeTests.cs index 90bfbf677..ac9117900 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceProbeTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceProbeTests.cs @@ -27,6 +27,6 @@ public sealed class DenoRuntimeTraceProbeTests Assert.Equal(new[] { "https://deno.land" }, metadata.RemoteOrigins); Assert.Equal(new[] { "fs", "net" }, metadata.UniquePermissions); - Assert.Equal("8f67e4b77f2ea4155d9101c5e6a45922e4ac1e19006955c3e6c2afe1938f0a8d", hash); + Assert.Equal("97f26acf896f0c2da77079885f6462cc7b589597b505532b09ae4bc6d1c0f314", hash); } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceRecorderTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceRecorderTests.cs index 7af6a450d..6f48a5112 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceRecorderTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceRecorderTests.cs @@ -45,13 +45,13 @@ public sealed class DenoRuntimeTraceRecorderTests Assert.Equal(2, snapshot.Metadata.ModuleLoads); Assert.Equal(1, snapshot.Metadata.PermissionUses); Assert.Equal(new[] { "https://deno.land/x/std" }, snapshot.Metadata.RemoteOrigins); - Assert.Equal(new[] { "net" }, snapshot.Metadata.UniquePermissions); + Assert.Equal(new[] { "fs", "net" }, snapshot.Metadata.UniquePermissions); Assert.Equal(0, snapshot.Metadata.NpmResolutions); Assert.Equal(0, snapshot.Metadata.WasmLoads); Assert.Equal(1, snapshot.Metadata.DynamicImports); // Stable hash check - Assert.Equal("198c6e038f1c39a78a52b844f051bfa6eaa5312faa66f1bc73d2f6d1048d8a7a", snapshot.Sha256); + Assert.Equal("61584731fc2870d972c78a86cb307d6f6dc5e110473d61b5bf61208f8be55e7a", snapshot.Sha256); } finally { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceSerializerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceSerializerTests.cs index 2bb44d44e..94161ee8c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceSerializerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Deno/DenoRuntimeTraceSerializerTests.cs @@ -46,6 +46,6 @@ public sealed class DenoRuntimeTraceSerializerTests "; Assert.Equal(expectedNdjson.Replace("\r\n", "\n"), text.Replace("\r\n", "\n")); - Assert.Equal("fdc6f07fe6b18b4cdd228c44b83e61d63063b7bd3422a2d3ab8000ac8420ceb0", hash); + Assert.Equal("9e74e46f576beafcfe76cd33b2f2f207bd2f1ba3cc86e045383c1afd52134961", hash); } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Golden/DenoAnalyzerGoldenTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Golden/DenoAnalyzerGoldenTests.cs index 97a877fc4..0afdb0c38 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Golden/DenoAnalyzerGoldenTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/Golden/DenoAnalyzerGoldenTests.cs @@ -32,6 +32,9 @@ public sealed class DenoAnalyzerGoldenTests normalized = normalized.TrimEnd(); expected = expected.TrimEnd(); + normalized = normalized.Replace("\r\n", "\n", StringComparison.Ordinal).Replace("\r", "\n", StringComparison.Ordinal); + expected = expected.Replace("\r\n", "\n", StringComparison.Ordinal).Replace("\r", "\n", StringComparison.Ordinal); + if (!string.Equals(expected, normalized, StringComparison.Ordinal)) { var actualPath = golden + ".actual"; @@ -201,9 +204,21 @@ public sealed class DenoAnalyzerGoldenTests var altRoot = workspaceRoot.Replace("/", "\\", StringComparison.Ordinal); var altRootLower = altRoot.ToLowerInvariant(); + var altRootEscaped = altRoot.Replace("\\", "\\\\", StringComparison.Ordinal); + var altRootLowerEscaped = altRootLower.Replace("\\", "\\\\", StringComparison.Ordinal); + var altRootDoubleEscaped = altRootEscaped.Replace("\\", "\\\\", StringComparison.Ordinal); + var altRootLowerDoubleEscaped = altRootLowerEscaped.Replace("\\", "\\\\", StringComparison.Ordinal); result = result .Replace(altRoot, "", StringComparison.Ordinal) - .Replace(altRootLower, "", StringComparison.Ordinal); + .Replace(altRootLower, "", StringComparison.Ordinal) + .Replace(altRootEscaped, "", StringComparison.Ordinal) + .Replace(altRootLowerEscaped, "", StringComparison.Ordinal) + .Replace(altRootDoubleEscaped, "", StringComparison.Ordinal) + .Replace(altRootLowerDoubleEscaped, "", StringComparison.Ordinal); + + result = result + .Replace("\\\\\\\\", "/", StringComparison.Ordinal) + .Replace("\\\\", "/", StringComparison.Ordinal); return result; } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests.csproj index a4e9da766..c34a8ead4 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests/StellaOps.Scanner.Analyzers.Lang.Deno.Tests.csproj @@ -14,12 +14,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests.csproj index 0709382d3..3c031079c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests/StellaOps.Scanner.Analyzers.Lang.DotNet.Tests.csproj @@ -6,7 +6,7 @@ enable false false - + false @@ -16,7 +16,6 @@ - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/Fixtures/lang/go/stripped/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/Fixtures/lang/go/stripped/expected.json index d93759843..048dc8b63 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/Fixtures/lang/go/stripped/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/Fixtures/lang/go/stripped/expected.json @@ -1,24 +1,24 @@ [ - { - "analyzerId": "golang", - "componentKey": "golang::bin::sha256:7125d65230b913faa744a33acd884899c81a1dbc6d88cbf251a74b19621cde99", - "name": "app", - "type": "bin", - "usedByEntrypoint": false, - "metadata": { - "binary.sha256": "7125d65230b913faa744a33acd884899c81a1dbc6d88cbf251a74b19621cde99", - "binaryPath": "app", - "go.version.hint": "go1.22.8", - "languageHint": "golang", - "provenance": "binary" + { + "analyzerId": "golang", + "componentKey": "golang::bin::sha256:80f528c90b72a4c4cc3fa078501154e4f2a3f49faea3ec380112d61740bde4c3", + "name": "app", + "type": "bin", + "usedByEntrypoint": false, + "metadata": { + "binary.sha256": "80f528c90b72a4c4cc3fa078501154e4f2a3f49faea3ec380112d61740bde4c3", + "binaryPath": "app", + "go.version.hint": "go1.22.8", + "languageHint": "golang", + "provenance": "binary" }, "evidence": [ { - "kind": "file", - "source": "binary", - "locator": "app", - "sha256": "7125d65230b913faa744a33acd884899c81a1dbc6d88cbf251a74b19621cde99" - }, + "kind": "file", + "source": "binary", + "locator": "app", + "sha256": "80f528c90b72a4c4cc3fa078501154e4f2a3f49faea3ec380112d61740bde4c3" + }, { "kind": "metadata", "source": "go.heuristic", diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests.csproj index e6f07dd30..9e4713966 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests/StellaOps.Scanner.Analyzers.Lang.Go.Tests.csproj @@ -14,12 +14,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/pomxml-only-jar/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/pomxml-only-jar/expected.json new file mode 100644 index 000000000..af9295ee3 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/pomxml-only-jar/expected.json @@ -0,0 +1,35 @@ +[ + { + "analyzerId": "java", + "componentKey": "purl::pkg:maven/com/example/pomxml-only@1.2.3", + "purl": "pkg:maven/com/example/pomxml-only@1.2.3", + "name": "pomxml-only", + "version": "1.2.3", + "type": "maven", + "usedByEntrypoint": true, + "metadata": { + "artifactId": "pomxml-only", + "displayName": "PomXml Only", + "groupId": "com.example", + "jarPath": "libs/pomxml-only.jar", + "manifestTitle": "PomXml Only", + "manifestVendor": "Example Corp", + "manifestVersion": "1.2.3", + "packaging": "jar" + }, + "evidence": [ + { + "kind": "file", + "source": "MANIFEST.MF", + "locator": "libs/pomxml-only.jar!META-INF/MANIFEST.MF", + "value": "title=PomXml Only;version=1.2.3;vendor=Example Corp" + }, + { + "kind": "file", + "source": "pom.xml", + "locator": "libs/pomxml-only.jar!META-INF/maven/com.example/pomxml-only/pom.xml", + "sha256": "9a315451470e76bb25c2a77ecdf03982aed210f1cbccab480c79eb1d4a5a79a5" + } + ] + } +] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/spring-boot-fat-embedded-maven/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/spring-boot-fat-embedded-maven/expected.json new file mode 100644 index 000000000..cc4e8e74e --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/spring-boot-fat-embedded-maven/expected.json @@ -0,0 +1,65 @@ +[ + { + "analyzerId": "java", + "componentKey": "purl::pkg:maven/com/example/app-fat@1.0.0", + "purl": "pkg:maven/com/example/app-fat@1.0.0", + "name": "app-fat", + "version": "1.0.0", + "type": "maven", + "usedByEntrypoint": true, + "metadata": { + "artifactId": "app-fat", + "displayName": "App Fat", + "embeddedScan.candidateJars": "1", + "embeddedScan.emittedComponents": "1", + "embeddedScan.scannedJars": "1", + "groupId": "com.example", + "jarPath": "apps/app-fat.jar", + "manifestTitle": "App Fat", + "manifestVendor": "Example Corp", + "manifestVersion": "1.0.0", + "packaging": "jar" + }, + "evidence": [ + { + "kind": "file", + "source": "MANIFEST.MF", + "locator": "apps/app-fat.jar!META-INF/MANIFEST.MF", + "value": "title=App Fat;version=1.0.0;vendor=Example Corp" + }, + { + "kind": "file", + "source": "pom.properties", + "locator": "apps/app-fat.jar!META-INF/maven/com.example/app-fat/pom.properties", + "sha256": "bba5da43d59efe9726f4195a86581d53b01bd449603fd2536fab29d720dcb806" + } + ] + }, + { + "analyzerId": "java", + "componentKey": "purl::pkg:maven/com/example/embedded-lib@2.1.0", + "purl": "pkg:maven/com/example/embedded-lib@2.1.0", + "name": "embedded-lib", + "version": "2.1.0", + "type": "maven", + "usedByEntrypoint": true, + "metadata": { + "artifactId": "embedded-lib", + "displayName": "Embedded Lib", + "embedded": "true", + "embedded.containerJarPath": "apps/app-fat.jar", + "embedded.entryPath": "BOOT-INF/lib/embedded-lib.jar", + "groupId": "com.example", + "jarPath": "apps/app-fat.jar!BOOT-INF/lib/embedded-lib.jar", + "packaging": "jar" + }, + "evidence": [ + { + "kind": "file", + "source": "pom.properties", + "locator": "apps/app-fat.jar!BOOT-INF/lib/embedded-lib.jar!META-INF/maven/com.example/embedded-lib/pom.properties", + "sha256": "45cbc64bcc2dcf25ee71a698cd35a676d79b0ed09cc77b61fead907c6345081f" + } + ] + } +] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/war-embedded-maven/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/war-embedded-maven/expected.json new file mode 100644 index 000000000..673312c99 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Fixtures/java/war-embedded-maven/expected.json @@ -0,0 +1,65 @@ +[ + { + "analyzerId": "java", + "componentKey": "purl::pkg:maven/com/example/demo-war@1.0.0?type=war", + "purl": "pkg:maven/com/example/demo-war@1.0.0?type=war", + "name": "demo-war", + "version": "1.0.0", + "type": "maven", + "usedByEntrypoint": true, + "metadata": { + "artifactId": "demo-war", + "displayName": "Demo War", + "embeddedScan.candidateJars": "1", + "embeddedScan.emittedComponents": "1", + "embeddedScan.scannedJars": "1", + "groupId": "com.example", + "jarPath": "apps/demo-war.war", + "manifestTitle": "Demo War", + "manifestVendor": "Example Corp", + "manifestVersion": "1.0.0", + "packaging": "war" + }, + "evidence": [ + { + "kind": "file", + "source": "MANIFEST.MF", + "locator": "apps/demo-war.war!META-INF/MANIFEST.MF", + "value": "title=Demo War;version=1.0.0;vendor=Example Corp" + }, + { + "kind": "file", + "source": "pom.properties", + "locator": "apps/demo-war.war!META-INF/maven/com.example/demo-war/pom.properties", + "sha256": "cb57c79ca5007119bfb0fafd6ae24a6702e508116d5e799835392df742a49460" + } + ] + }, + { + "analyzerId": "java", + "componentKey": "purl::pkg:maven/com/example/web-lib@3.0.0", + "purl": "pkg:maven/com/example/web-lib@3.0.0", + "name": "web-lib", + "version": "3.0.0", + "type": "maven", + "usedByEntrypoint": true, + "metadata": { + "artifactId": "web-lib", + "displayName": "Web Lib", + "embedded": "true", + "embedded.containerJarPath": "apps/demo-war.war", + "embedded.entryPath": "WEB-INF/lib/web-lib.jar", + "groupId": "com.example", + "jarPath": "apps/demo-war.war!WEB-INF/lib/web-lib.jar", + "packaging": "jar" + }, + "evidence": [ + { + "kind": "file", + "source": "pom.properties", + "locator": "apps/demo-war.war!WEB-INF/lib/web-lib.jar!META-INF/maven/com.example/web-lib/pom.properties", + "sha256": "ab7151e977ef21d48c459395dbb0f88395a3a33b2f5903a28b7d78b53cb8880d" + } + ] + } +] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Java/JavaLanguageAnalyzerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Java/JavaLanguageAnalyzerTests.cs index dfa8ffe01..7a3a31152 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Java/JavaLanguageAnalyzerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/Java/JavaLanguageAnalyzerTests.cs @@ -36,6 +36,81 @@ public sealed class JavaLanguageAnalyzerTests } } + [Fact] + public async Task ExtractsMavenArtifactsFromSpringBootFatJarEmbeddedLibrariesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + try + { + var jarPath = JavaFixtureBuilder.CreateSpringBootFatJarWithEmbeddedMavenLibrary(root); + var usageHints = new LanguageUsageHints(new[] { jarPath }); + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var goldenPath = TestPaths.ResolveFixture("java", "spring-boot-fat-embedded-maven", "expected.json"); + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath: root, + goldenPath: goldenPath, + analyzers: analyzers, + cancellationToken: cancellationToken, + usageHints: usageHints); + } + finally + { + TestPaths.SafeDelete(root); + } + } + + [Fact] + public async Task ExtractsMavenArtifactsFromWarEmbeddedLibrariesAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + try + { + var warPath = JavaFixtureBuilder.CreateWarArchiveWithEmbeddedMavenLibrary(root); + var usageHints = new LanguageUsageHints(new[] { warPath }); + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var goldenPath = TestPaths.ResolveFixture("java", "war-embedded-maven", "expected.json"); + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath: root, + goldenPath: goldenPath, + analyzers: analyzers, + cancellationToken: cancellationToken, + usageHints: usageHints); + } + finally + { + TestPaths.SafeDelete(root); + } + } + + [Fact] + public async Task ExtractsMavenArtifactsFromPomXmlOnlyJarAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + try + { + var jarPath = JavaFixtureBuilder.CreatePomXmlOnlyJar(root); + var usageHints = new LanguageUsageHints(new[] { jarPath }); + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var goldenPath = TestPaths.ResolveFixture("java", "pomxml-only-jar", "expected.json"); + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath: root, + goldenPath: goldenPath, + analyzers: analyzers, + cancellationToken: cancellationToken, + usageHints: usageHints); + } + finally + { + TestPaths.SafeDelete(root); + } + } + [Fact] public async Task LockfilesProduceDeclaredOnlyComponentsAsync() { @@ -157,7 +232,12 @@ public sealed class JavaLanguageAnalyzerTests WritePomProperties(archive, "com.example", "demo-jni", "1.0.0"); WriteManifest(archive, "demo-jni", "1.0.0", "com.example"); - CreateBinaryEntry(archive, "com/example/App.class", "System.loadLibrary(\"foo\")"); + var classEntry = archive.CreateEntry("com/example/App.class"); + var classBytes = JavaClassFileFactory.CreateSystemLoadLibraryInvoker("com/example/App", "foo"); + using (var classStream = classEntry.Open()) + { + classStream.Write(classBytes); + } CreateTextEntry(archive, "lib/native/libfoo.so"); CreateTextEntry(archive, "META-INF/native-image/demo/jni-config.json"); } @@ -177,7 +257,197 @@ public sealed class JavaLanguageAnalyzerTests var metadata = component.GetProperty("metadata"); Assert.Equal("libfoo.so", metadata.GetProperty("jni.nativeLibs").GetString()); Assert.Equal("demo-jni.jar!META-INF/native-image/demo/jni-config.json", metadata.GetProperty("jni.graalConfig").GetString()); - Assert.Equal("demo-jni.jar!com/example/App.class", metadata.GetProperty("jni.loadCalls").GetString()); + Assert.Equal("1", metadata.GetProperty("jni.edgeCount").GetString()); + Assert.Equal("0", metadata.GetProperty("jni.nativeMethodCount").GetString()); + Assert.Equal("1", metadata.GetProperty("jni.loadCallCount").GetString()); + Assert.Equal("SystemLoadLibrary", metadata.GetProperty("jni.reasons").GetString()); + Assert.Equal("foo", metadata.GetProperty("jni.targetLibraries").GetString()); + } + finally + { + TestPaths.SafeDelete(root); + } + } + + [Fact] + public async Task ExtractsMavenArtifactFromEmbeddedJarAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + + try + { + var jarPath = Path.Combine(root, "demo-fat.jar"); + Directory.CreateDirectory(Path.GetDirectoryName(jarPath)!); + + using (var archive = ZipFile.Open(jarPath, ZipArchiveMode.Create)) + { + WritePomProperties(archive, "com.example", "demo-fat", "1.0.0"); + WriteManifest(archive, "demo-fat", "1.0.0", "com.example"); + + using var embeddedBuffer = new MemoryStream(); + using (var embeddedJar = new ZipArchive(embeddedBuffer, ZipArchiveMode.Create, leaveOpen: true)) + { + WritePomProperties(embeddedJar, "com.example", "embedded-lib", "2.0.0"); + } + + embeddedBuffer.Position = 0; + var embeddedEntry = archive.CreateEntry("BOOT-INF/lib/embedded-lib.jar"); + using var embeddedStream = embeddedEntry.Open(); + embeddedBuffer.CopyTo(embeddedStream); + } + + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( + root, + analyzers, + cancellationToken, + new LanguageUsageHints(new[] { jarPath })); + + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var outer = components.First(c => c.GetProperty("name").GetString() == "demo-fat"); + var outerMetadata = outer.GetProperty("metadata"); + Assert.Equal("1", outerMetadata.GetProperty("embeddedScan.candidateJars").GetString()); + Assert.Equal("1", outerMetadata.GetProperty("embeddedScan.scannedJars").GetString()); + Assert.Equal("1", outerMetadata.GetProperty("embeddedScan.emittedComponents").GetString()); + + var embedded = components.First(c => c.GetProperty("name").GetString() == "embedded-lib"); + var embeddedMetadata = embedded.GetProperty("metadata"); + Assert.Equal("true", embeddedMetadata.GetProperty("embedded").GetString()); + Assert.Equal("demo-fat.jar!BOOT-INF/lib/embedded-lib.jar", embeddedMetadata.GetProperty("jarPath").GetString()); + + var embeddedEvidence = embedded.GetProperty("evidence").EnumerateArray().ToArray(); + Assert.Contains(embeddedEvidence, e => + string.Equals(e.GetProperty("source").GetString(), "pom.properties", StringComparison.OrdinalIgnoreCase) && + string.Equals(e.GetProperty("locator").GetString(), "demo-fat.jar!BOOT-INF/lib/embedded-lib.jar!META-INF/maven/com.example/embedded-lib/pom.properties", StringComparison.OrdinalIgnoreCase) && + e.TryGetProperty("sha256", out var sha) && + !string.IsNullOrWhiteSpace(sha.GetString())); + } + finally + { + TestPaths.SafeDelete(root); + } + } + + [Fact] + public async Task ExtractsMavenArtifactFromPomXmlWhenPomPropertiesMissingAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + + try + { + var jarPath = Path.Combine(root, "demo-pomxml.jar"); + Directory.CreateDirectory(Path.GetDirectoryName(jarPath)!); + + using (var archive = ZipFile.Open(jarPath, ZipArchiveMode.Create)) + { + WriteManifest(archive, "demo-pomxml", "1.2.3", "com.example"); + + var pomXmlPath = "META-INF/maven/com.example/demo-pomxml/pom.xml"; + var pomXml = """ + + 4.0.0 + com.example + demo-pomxml + 1.2.3 + Demo Pom XML + + """; + CreateTextEntry(archive, pomXmlPath, pomXml); + } + + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( + root, + analyzers, + cancellationToken, + new LanguageUsageHints(new[] { jarPath })); + + using var document = JsonDocument.Parse(json); + var component = document.RootElement + .EnumerateArray() + .First(element => string.Equals(element.GetProperty("name").GetString(), "demo-pomxml", StringComparison.Ordinal)); + + Assert.Equal("pkg:maven/com/example/demo-pomxml@1.2.3", component.GetProperty("purl").GetString()); + + var evidence = component.GetProperty("evidence").EnumerateArray().ToArray(); + Assert.Contains(evidence, e => + string.Equals(e.GetProperty("source").GetString(), "pom.xml", StringComparison.OrdinalIgnoreCase) && + string.Equals(e.GetProperty("locator").GetString(), "demo-pomxml.jar!META-INF/maven/com.example/demo-pomxml/pom.xml", StringComparison.OrdinalIgnoreCase) && + e.TryGetProperty("sha256", out var sha) && + !string.IsNullOrWhiteSpace(sha.GetString())); + } + finally + { + TestPaths.SafeDelete(root); + } + } + + [Fact] + public async Task PomXmlWithIncompleteCoordinatesEmitsUnresolvedComponentAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + + try + { + var jarPath = Path.Combine(root, "demo-pomxml-unresolved.jar"); + Directory.CreateDirectory(Path.GetDirectoryName(jarPath)!); + + using (var archive = ZipFile.Open(jarPath, ZipArchiveMode.Create)) + { + WriteManifest(archive, "demo-pomxml-unresolved", "9.9.9", "com.example"); + + var pomXmlPath = "META-INF/maven/com.example/demo-pomxml-unresolved/pom.xml"; + var pomXml = """ + + 4.0.0 + com.example + demo-pomxml-unresolved + + """; + CreateTextEntry(archive, pomXmlPath, pomXml); + } + + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( + root, + analyzers, + cancellationToken, + new LanguageUsageHints(new[] { jarPath })); + + using var document = JsonDocument.Parse(json); + var component = document.RootElement + .EnumerateArray() + .First(element => + { + if (!element.TryGetProperty("metadata", out var metadata) || metadata.ValueKind != JsonValueKind.Object) + { + return false; + } + + return metadata.TryGetProperty("unresolvedCoordinates", out var unresolved) + && string.Equals(unresolved.GetString(), "true", StringComparison.Ordinal); + }); + + if (component.TryGetProperty("purl", out var purl)) + { + Assert.Equal(JsonValueKind.Null, purl.ValueKind); + } + + var metadata = component.GetProperty("metadata"); + Assert.Equal("demo-pomxml-unresolved", metadata.GetProperty("manifestTitle").GetString()); + Assert.Equal("9.9.9", metadata.GetProperty("manifestVersion").GetString()); + + var evidence = component.GetProperty("evidence").EnumerateArray().ToArray(); + Assert.Contains(evidence, e => + string.Equals(e.GetProperty("source").GetString(), "pom.xml", StringComparison.OrdinalIgnoreCase) && + string.Equals(e.GetProperty("locator").GetString(), "demo-pomxml-unresolved.jar!META-INF/maven/com.example/demo-pomxml-unresolved/pom.xml", StringComparison.OrdinalIgnoreCase) && + e.TryGetProperty("sha256", out var sha) && + !string.IsNullOrWhiteSpace(sha.GetString())); } finally { @@ -436,14 +706,6 @@ public sealed class JavaLanguageAnalyzerTests } } - private static void CreateBinaryEntry(ZipArchive archive, string path, string content) - { - var entry = archive.CreateEntry(path); - using var stream = entry.Open(); - var bytes = Encoding.UTF8.GetBytes(content); - stream.Write(bytes, 0, bytes.Length); - } - private static string CreateSampleJar(string root, string groupId, string artifactId, string version) { var jarPath = Path.Combine(root, $"{artifactId}-{version}.jar"); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests.csproj index 7d34c1cbb..786c0021b 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests/StellaOps.Scanner.Analyzers.Lang.Java.Tests.csproj @@ -14,12 +14,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/expected.json new file mode 100644 index 000000000..8dc709ccc --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/expected.json @@ -0,0 +1,80 @@ +[ + { + "analyzerId": "node-phase22", + "componentKey": "/app/native/addon.node", + "name": "addon.node", + "type": "node:native", + "usedByEntrypoint": false, + "metadata": { + "confidence": "0.82", + "reason": "native-addon-file" + }, + "evidence": [] + }, + { + "analyzerId": "node-phase22", + "componentKey": "/app/pkg/pkg.wasm", + "name": "pkg.wasm", + "type": "node:wasm", + "usedByEntrypoint": false, + "metadata": { + "confidence": "0.80", + "reason": "wasm-file" + }, + "evidence": [] + }, + { + "analyzerId": "node-phase22", + "componentKey": "/src/app.js", + "name": "app.js", + "type": "node:bundle", + "usedByEntrypoint": false, + "metadata": { + "confidence": "0.87", + "format": "esm", + "reason": "source-map" + }, + "evidence": [] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/cached-lib@1.0.0", + "purl": "pkg:npm/cached-lib@1.0.0", + "name": "cached-lib", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "path": ".yarn/cache", + "yarnPnp": "true" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": ".yarn/cache/cached-lib-1.0.0.zip!package/package.json", + "sha256": "b13d2a5d313d5929280c14af2086e23ca8f0d60761085c0ad44982ec307c92e3" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/yarn-pnp-demo@1.0.0", + "purl": "pkg:npm/yarn-pnp-demo@1.0.0", + "name": "yarn-pnp-demo", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "path": ".", + "yarnPnp": "true" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "package.json" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-env/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-env/expected.json index de190bbb6..2366d8cde 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-env/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-env/expected.json @@ -23,7 +23,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "4cd71adf540fff675b46ecd4d88d0b186534e97f9ca57ee86588d1386deb9274" } ] }, @@ -48,4 +49,4 @@ } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/expected.json index 5ce4bc989..d46949e1a 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/expected.json @@ -1,4 +1,51 @@ [ + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/hidden-lib@0.2.0", + "purl": "pkg:npm/hidden-lib@0.2.0", + "name": "hidden-lib", + "version": "0.2.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "entrypoint": ".layers/layer0/node_modules/hidden-lib/index.js", + "path": ".layers/layer0/node_modules/hidden-lib" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": ".layers/layer0/node_modules/hidden-lib/package.json", + "sha256": "d014f340282aa989e5887ddf10a1d2165ba556f89428d0f2812eb8ce8e63c1c8" + }, + { + "kind": "metadata", + "source": "package.json:entrypoint", + "locator": ".layers/layer0/node_modules/hidden-lib/package.json#entrypoint", + "value": ".layers/layer0/node_modules/hidden-lib/index.js;index.js" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/layer-app@0.1.0", + "purl": "pkg:npm/layer-app@0.1.0", + "name": "layer-app", + "version": "0.1.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "path": "layers/layer1/app" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "layers/layer1/app/package.json", + "sha256": "23abb943f062b3ccdc18966eb36dfc48dd7ec4b5a6105851484fe2911946ecdd" + } + ] + }, { "analyzerId": "node", "componentKey": "purl::pkg:npm/layer-lib@0.1.0", @@ -15,7 +62,8 @@ { "kind": "file", "source": "package.json", - "locator": "layers/layer1/node_modules/layer-lib/package.json" + "locator": "layers/layer1/node_modules/layer-lib/package.json", + "sha256": "4d4ee909c5fa810d7e9a1bb74f4e6e2da59c3bb4182f62f8bb8f6074218f19d8" }, { "kind": "metadata", @@ -24,5 +72,32 @@ "value": "layers/layer1/node_modules/layer-lib/index.js;index.js" } ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/top-layer-lib@0.3.0", + "purl": "pkg:npm/top-layer-lib@0.3.0", + "name": "top-layer-lib", + "version": "0.3.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "entrypoint": "layer2/node_modules/top-layer-lib/index.js", + "path": "layer2/node_modules/top-layer-lib" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "layer2/node_modules/top-layer-lib/package.json", + "sha256": "9de01a780c07e3d34ef74dfdbd14c5173f419609e77f9cc1fb973c30400e30e9" + }, + { + "kind": "metadata", + "source": "package.json:entrypoint", + "locator": "layer2/node_modules/top-layer-lib/package.json#entrypoint", + "value": "layer2/node_modules/top-layer-lib/index.js;index.js" + } + ] } ] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/layers/layer1/app/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/layers/layer1/app/package.json new file mode 100644 index 000000000..ff63d1be2 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/container-layers/layers/layer1/app/package.json @@ -0,0 +1,7 @@ +{ + "name": "layer-app", + "version": "0.1.0", + "dependencies": { + "layer-lib": "1.0.0" + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/expected.json new file mode 100644 index 000000000..db9018a56 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/expected.json @@ -0,0 +1,200 @@ +[ + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::dev-range::sha256:681581098c1c40eb1faebe65e1916e56011369708306debd2568bfaa8173a6fc", + "name": "dev-range", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#devDependencies", + "declared.scope": "development", + "declared.source": "package.json", + "declared.sourceType": "range", + "declared.versionSpec": "~2.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#devDependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::file-lib::sha256:2222fa2f2ec7523cabd6c80a0ffd89c3b306fad0ff6b89856f8b896e0b2fe70e", + "name": "file-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "production", + "declared.source": "package.json", + "declared.sourceType": "file", + "declared.versionSpec": "file:../local/file-lib", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#dependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::git-lib::sha256:02362d8e5c76a43f5f657721c46db1370be1c5504c9b30be8f667d5d85f369c6", + "name": "git-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "production", + "declared.source": "package.json", + "declared.sourceType": "git", + "declared.versionSpec": "git\u002Bhttps://example.com/repo.git#v1.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#dependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::opt-lib::sha256:79ae0f26f0d9e4a55710b56ae3f216251cedb886ce9dd45891fd70a17d2c273b", + "name": "opt-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#optionalDependencies", + "declared.scope": "optional", + "declared.source": "package.json", + "declared.sourceType": "workspace", + "declared.versionSpec": "workspace:*", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#optionalDependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::path-lib::sha256:5dfa2181ae287045798e7f7d1b3d8f452f071f19be8366d0a218da64fe815589", + "name": "path-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "production", + "declared.source": "package.json", + "declared.sourceType": "path", + "declared.versionSpec": "./local/path-lib", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#dependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::peer-lib::sha256:518814536b4014ed1645c41ccf2de9349336f33be755dfe30d60b14f85f5b302", + "name": "peer-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#peerDependencies", + "declared.scope": "peer", + "declared.source": "package.json", + "declared.sourceType": "range", + "declared.versionSpec": "\u003E=3.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#peerDependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::range-lib::sha256:d90425e51e919a0e90bd2cae825d86919c79799ed49ce2ecdeb956de86344145", + "name": "range-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "production", + "declared.source": "package.json", + "declared.sourceType": "range", + "declared.versionSpec": "^1.2.3", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#dependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::tag-lib::sha256:8337331b8a282de0437b980185f47c0d4f592b79d1b114a67b5dad11b8815afd", + "name": "tag-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "production", + "declared.source": "package.json", + "declared.sourceType": "tag", + "declared.versionSpec": "latest", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#dependencies" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::tarball-lib::sha256:4fcba99e507cd649bf51963a3ed4f16fbf6431a4170e08580495999ad1ea1fd6", + "name": "tarball-lib", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package.json#dependencies", + "declared.scope": "production", + "declared.source": "package.json", + "declared.sourceType": "tarball", + "declared.versionSpec": "https://example.com/pkg.tgz", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package.json#dependencies" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/package.json new file mode 100644 index 000000000..1d388990a --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/declared-only-package-json/package.json @@ -0,0 +1,20 @@ +{ + "private": true, + "dependencies": { + "range-lib": "^1.2.3", + "tag-lib": "latest", + "git-lib": "git+https://example.com/repo.git#v1.0.0", + "tarball-lib": "https://example.com/pkg.tgz", + "file-lib": "file:../local/file-lib", + "path-lib": "./local/path-lib" + }, + "devDependencies": { + "dev-range": "~2.0.0" + }, + "peerDependencies": { + "peer-lib": ">=3.0.0" + }, + "optionalDependencies": { + "opt-lib": "workspace:*" + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/entrypoints/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/entrypoints/expected.json index 3297957f4..f5d054d15 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/entrypoints/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/entrypoints/expected.json @@ -16,7 +16,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "e20b4c9ec9073b572c368b5ea40465eb59586487fa9469ae784cc23f618f3457" }, { "kind": "metadata", diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/imports-dynamic/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/imports-dynamic/expected.json index cbf374a87..9f6e8c6dd 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/imports-dynamic/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/imports-dynamic/expected.json @@ -30,13 +30,63 @@ "usedByEntrypoint": false, "metadata": { "entrypoint": "src/index.js", + "imports": "4", "path": "." }, "evidence": [ + { + "kind": "file", + "source": "node.import", + "locator": "src/index.js", + "value": "./lib//entry.js${*} (conf:medium;template-dynamic)" + }, + { + "kind": "file", + "source": "node.import", + "locator": "src/index.js", + "value": "./lib/concat.js (conf:high;literal;literal)" + }, + { + "kind": "file", + "source": "node.import", + "locator": "src/index.js", + "value": "./lib/static.js (conf:high;literal)" + }, + { + "kind": "file", + "source": "node.import", + "locator": "src/original.ts", + "value": "./lib/sourcemap.js (conf:high;literal)" + }, { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "07f1225926d9d07b0a024994036d10a689f3d98cc51324e2b21a06a7bddb8d0e" + }, + { + "kind": "metadata", + "source": "node.resolve", + "locator": "package.json", + "value": "src/index.js:./lib//entry.js${*}-\u003Eunresolved (unresolved;low)" + }, + { + "kind": "metadata", + "source": "node.resolve", + "locator": "package.json", + "value": "src/index.js:./lib/concat.js-\u003Eunresolved (unresolved;low)" + }, + { + "kind": "metadata", + "source": "node.resolve", + "locator": "package.json", + "value": "src/index.js:./lib/static.js-\u003Eunresolved (unresolved;low)" + }, + { + "kind": "metadata", + "source": "node.resolve", + "locator": "package.json", + "value": "src/original.ts:./lib/sourcemap.js-\u003Eunresolved (unresolved;low)" }, { "kind": "metadata", diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/expected.json new file mode 100644 index 000000000..060e510d0 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/expected.json @@ -0,0 +1,98 @@ +[ + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/%40scope/scoped-child@4.0.0", + "purl": "pkg:npm/%40scope/scoped-child@4.0.0", + "name": "@scope/scoped-child", + "version": "4.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package-lock.json:node_modules/parent/node_modules/@scope/scoped-child", + "declared.resolvedVersion": "4.0.0", + "declared.source": "package-lock.json", + "declared.sourceType": "range", + "declared.versionSpec": "4.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package-lock.json:node_modules/parent/node_modules/@scope/scoped-child" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/%40scope/scoped@3.0.0", + "purl": "pkg:npm/%40scope/scoped@3.0.0", + "name": "@scope/scoped", + "version": "3.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package-lock.json:node_modules/@scope/scoped", + "declared.resolvedVersion": "3.0.0", + "declared.source": "package-lock.json", + "declared.sourceType": "range", + "declared.versionSpec": "3.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package-lock.json:node_modules/@scope/scoped" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/child@2.0.0", + "purl": "pkg:npm/child@2.0.0", + "name": "child", + "version": "2.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package-lock.json:node_modules/parent/node_modules/child", + "declared.resolvedVersion": "2.0.0", + "declared.source": "package-lock.json", + "declared.sourceType": "range", + "declared.versionSpec": "2.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package-lock.json:node_modules/parent/node_modules/child" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/parent@1.0.0", + "purl": "pkg:npm/parent@1.0.0", + "name": "parent", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package-lock.json:node_modules/parent", + "declared.resolvedVersion": "1.0.0", + "declared.source": "package-lock.json", + "declared.sourceType": "range", + "declared.versionSpec": "1.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package-lock.json:node_modules/parent" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/package-lock.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/package-lock.json new file mode 100644 index 000000000..4c0589a7d --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-package-lock/package-lock.json @@ -0,0 +1,18 @@ +{ + "name": "lock-only-package-lock", + "lockfileVersion": 3, + "packages": { + "node_modules/parent": { + "version": "1.0.0" + }, + "node_modules/parent/node_modules/child": { + "version": "2.0.0" + }, + "node_modules/@scope/scoped": { + "version": "3.0.0" + }, + "node_modules/parent/node_modules/@scope/scoped-child": { + "version": "4.0.0" + } + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/expected.json new file mode 100644 index 000000000..df19ba502 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/expected.json @@ -0,0 +1,75 @@ +[ + { + "analyzerId": "node", + "componentKey": "explicit::node::npm::local-link::sha256:ea69a4c271be378e84e910ddee0a83c3b6da4b4526146ce646e3ce411ae0aa07", + "name": "local-link", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "pnpm-lock.yaml:local-link/0.0.0", + "declared.source": "pnpm-lock.yaml", + "declared.sourceType": "link", + "declared.versionSpec": "link:../packages/local-link", + "declaredOnly": "true", + "lockIntegrityMissing": "true", + "lockIntegrityMissingReason": "link" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "pnpm-lock.yaml:local-link/0.0.0" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/%40scope/scoped@2.0.0", + "purl": "pkg:npm/%40scope/scoped@2.0.0", + "name": "@scope/scoped", + "version": "2.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "pnpm-lock.yaml:@scope/scoped/2.0.0", + "declared.resolvedVersion": "2.0.0", + "declared.source": "pnpm-lock.yaml", + "declared.sourceType": "range", + "declared.versionSpec": "2.0.0", + "declaredOnly": "true", + "lockIntegrityMissing": "true", + "lockIntegrityMissingReason": "missing" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "pnpm-lock.yaml:@scope/scoped/2.0.0" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/left-pad@1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "name": "left-pad", + "version": "1.3.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "pnpm-lock.yaml:left-pad/1.3.0", + "declared.resolvedVersion": "1.3.0", + "declared.source": "pnpm-lock.yaml", + "declared.sourceType": "range", + "declared.versionSpec": "1.3.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "pnpm-lock.yaml:left-pad/1.3.0" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/pnpm-lock.yaml b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/pnpm-lock.yaml new file mode 100644 index 000000000..78d21b84b --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-pnpm/pnpm-lock.yaml @@ -0,0 +1,10 @@ +lockfileVersion: 6.0 + +packages: + /left-pad/1.3.0: + resolution: {integrity: sha512-leftpad, tarball: https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz} + /local-link/0.0.0: + version: link:../packages/local-link + resolution: {tarball: link:../packages/local-link} + /@scope/scoped/2.0.0: + resolution: {tarball: https://registry.npmjs.org/@scope/scoped/-/scoped-2.0.0.tgz} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/expected.json new file mode 100644 index 000000000..ad06ce6f1 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/expected.json @@ -0,0 +1,98 @@ +[ + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/%40scope/scoped@3.0.1", + "purl": "pkg:npm/%40scope/scoped@3.0.1", + "name": "@scope/scoped", + "version": "3.0.1", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "yarn.lock:@scope/scoped@npm:^3.0.0", + "declared.resolvedVersion": "3.0.1", + "declared.source": "yarn.lock", + "declared.sourceType": "range", + "declared.versionSpec": "3.0.1", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "yarn.lock:@scope/scoped@npm:^3.0.0" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/legacy@1.0.0", + "purl": "pkg:npm/legacy@1.0.0", + "name": "legacy", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "yarn.lock:legacy@^1.0.0", + "declared.resolvedVersion": "1.0.0", + "declared.source": "yarn.lock", + "declared.sourceType": "range", + "declared.versionSpec": "1.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "yarn.lock:legacy@^1.0.0" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/multi@1.0.0", + "purl": "pkg:npm/multi@1.0.0", + "name": "multi", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "yarn.lock:multi@npm:^1.0.0", + "declared.resolvedVersion": "1.0.0", + "declared.source": "yarn.lock", + "declared.sourceType": "range", + "declared.versionSpec": "1.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "yarn.lock:multi@npm:^1.0.0" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/multi@2.0.0", + "purl": "pkg:npm/multi@2.0.0", + "name": "multi", + "version": "2.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "yarn.lock:multi@npm:^2.0.0", + "declared.resolvedVersion": "2.0.0", + "declared.source": "yarn.lock", + "declared.sourceType": "range", + "declared.versionSpec": "2.0.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "yarn.lock:multi@npm:^2.0.0" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/yarn.lock b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/yarn.lock new file mode 100644 index 000000000..6deb16827 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/lock-only-yarn-berry/yarn.lock @@ -0,0 +1,23 @@ +__metadata: + version: 8 + cacheKey: 10 + +"multi@npm:^1.0.0": + version: "1.0.0" + resolution: "multi@npm:1.0.0" + checksum: "abcd1234" + +"multi@npm:^2.0.0": + version: "2.0.0" + resolution: "multi@npm:2.0.0" + integrity: "sha512-xyz987" + +"@scope/scoped@npm:^3.0.0": + version: "3.0.1" + resolution: "@scope/scoped@npm:3.0.1" + checksum: "deadbeef" + +legacy@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/legacy/-/legacy-1.0.0.tgz#abc123" + integrity sha512-legacy diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/pnpm-store/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/pnpm-store/expected.json index d9c4b91a3..895084163 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/pnpm-store/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/pnpm-store/expected.json @@ -17,7 +17,8 @@ { "kind": "file", "source": "package.json", - "locator": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg/package.json" + "locator": "node_modules/.pnpm/pkg@1.2.3/node_modules/pkg/package.json", + "sha256": "23fc3dc23387b21780c9b0b3f8cf3e07e1619a0603325a2744e2a6d2873fceac" }, { "kind": "metadata", @@ -42,7 +43,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "1ee30a64ac1806fececa1a00c36555930029c81c92de4b376c2610499c1cb435" } ] } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/runtime-evidence/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/runtime-evidence/expected.json index b37e4ca5b..1154cc258 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/runtime-evidence/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/runtime-evidence/expected.json @@ -34,7 +34,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "e7bea1ac14004d809d1b649c1329c9e09ce69d458c3794ee98b4f37c6fa591b5" }, { "kind": "metadata", diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/shebang/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/shebang/expected.json index d0fe3f01e..906232918 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/shebang/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/shebang/expected.json @@ -16,7 +16,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "452a5c537c19282754f6c32eebf8aea46e9604c76b8d3b16527cdab932701ff7" }, { "kind": "metadata", diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/version-targets/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/version-targets/expected.json index beec84f84..983a16352 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/version-targets/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/version-targets/expected.json @@ -31,8 +31,9 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "3c16561eea74166e8add2e59d2bf93d55c035a7cdac640410332ee2b9e7a1a35" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/expected.json index 736a9544a..c78f2bb7b 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/expected.json @@ -1,4 +1,53 @@ [ + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/declared-only@9.9.9", + "purl": "pkg:npm/declared-only@9.9.9", + "name": "declared-only", + "version": "9.9.9", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package-lock.json:packages/app/node_modules/declared-only", + "declared.resolvedVersion": "9.9.9", + "declared.source": "package-lock.json", + "declared.sourceType": "range", + "declared.versionSpec": "9.9.9", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package-lock.json:packages/app/node_modules/declared-only" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/left-pad@1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "name": "left-pad", + "version": "1.3.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "declared.locator": "package-lock.json:packages/app/node_modules/left-pad", + "declared.resolvedVersion": "1.3.0", + "declared.scope": "production", + "declared.source": "package-lock.json", + "declared.sourceType": "range", + "declared.versionSpec": "1.3.0", + "declaredOnly": "true" + }, + "evidence": [ + { + "kind": "metadata", + "source": "node.declared", + "locator": "package-lock.json:packages/app/node_modules/left-pad" + } + ] + }, { "analyzerId": "node", "componentKey": "purl::pkg:npm/lib@2.0.1", @@ -13,6 +62,8 @@ "lockSource": "package-lock.json", "path": "packages/lib", "resolved": "https://registry.example/lib-2.0.1.tgz", + "riskLevel": "production", + "scope": "production", "workspaceMember": "true", "workspaceRoot": "packages/lib" }, @@ -20,7 +71,30 @@ { "kind": "file", "source": "package.json", - "locator": "packages/lib/package.json" + "locator": "packages/lib/package.json", + "sha256": "5198fbaf659fb4f8d9845a7ffa51067ac7e727631f2503615c962540fe8c2298" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/nested-tool@0.0.5", + "purl": "pkg:npm/nested-tool@0.0.5", + "name": "nested-tool", + "version": "0.0.5", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "path": "packages/nested/tool", + "workspaceMember": "true", + "workspaceRoot": "packages/nested/tool" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "packages/nested/tool/package.json", + "sha256": "3011f57f07fab11b4ecb61788319bc9768d2577cafd9f53f37a7cac721fc77cf" } ] }, @@ -42,7 +116,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json" + "locator": "package.json", + "sha256": "aa060a0c2a8a6c41f68783d0f7366491e5560bb9af3ea043d6d2dc664de20a7f" } ] }, @@ -60,6 +135,8 @@ "lockSource": "package-lock.json", "path": "packages/shared", "resolved": "https://registry.example/shared-3.1.4.tgz", + "riskLevel": "production", + "scope": "production", "workspaceMember": "true", "workspaceRoot": "packages/shared", "workspaceTargets": "packages/lib" @@ -68,7 +145,8 @@ { "kind": "file", "source": "package.json", - "locator": "packages/shared/package.json" + "locator": "packages/shared/package.json", + "sha256": "4440d351c91132499bedb21859b2a1813b25563b93a54eb1ad1ded79d18839d1" } ] }, @@ -95,7 +173,8 @@ { "kind": "file", "source": "package.json", - "locator": "packages/app/package.json" + "locator": "packages/app/package.json", + "sha256": "e734fc2024e5582200309bd190832f2f931a8b8af65dd11c49f0583c92195582" }, { "kind": "metadata", @@ -106,4 +185,4 @@ } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/package.json index 03fded30f..caf8f117c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/package.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/package.json @@ -1,10 +1,9 @@ { "name": "root-workspace", - "version": "1.0.0", - "private": true, - "workspaces": [ - "packages/app", - "packages/lib", - "packages/shared" - ] -} + "version": "1.0.0", + "private": true, + "workspaces": [ + "packages/*", + "packages/**" + ] +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/packages/nested/tool/package.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/packages/nested/tool/package.json new file mode 100644 index 000000000..78900fb9b --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/workspaces/packages/nested/tool/package.json @@ -0,0 +1,7 @@ +{ + "name": "nested-tool", + "version": "0.0.5", + "devDependencies": { + "left-pad": "1.3.0" + } +} diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/yarn-pnp/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/yarn-pnp/expected.json index 57c6fd105..8dc709ccc 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/yarn-pnp/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Fixtures/lang/node/yarn-pnp/expected.json @@ -1,4 +1,41 @@ [ + { + "analyzerId": "node-phase22", + "componentKey": "/app/native/addon.node", + "name": "addon.node", + "type": "node:native", + "usedByEntrypoint": false, + "metadata": { + "confidence": "0.82", + "reason": "native-addon-file" + }, + "evidence": [] + }, + { + "analyzerId": "node-phase22", + "componentKey": "/app/pkg/pkg.wasm", + "name": "pkg.wasm", + "type": "node:wasm", + "usedByEntrypoint": false, + "metadata": { + "confidence": "0.80", + "reason": "wasm-file" + }, + "evidence": [] + }, + { + "analyzerId": "node-phase22", + "componentKey": "/src/app.js", + "name": "app.js", + "type": "node:bundle", + "usedByEntrypoint": false, + "metadata": { + "confidence": "0.87", + "format": "esm", + "reason": "source-map" + }, + "evidence": [] + }, { "analyzerId": "node", "componentKey": "purl::pkg:npm/cached-lib@1.0.0", @@ -8,9 +45,7 @@ "type": "npm", "usedByEntrypoint": false, "metadata": { - "lockLocator": "cached-lib@npm:1.0.0", - "lockSource": "pnp.data", - "path": ".yarn/cache/cached-lib-1.0.0.zip/node_modules/cached-lib", + "path": ".yarn/cache", "yarnPnp": "true" }, "evidence": [ @@ -31,8 +66,6 @@ "type": "npm", "usedByEntrypoint": false, "metadata": { - "lockLocator": "yarn-pnp-demo@workspace:.", - "lockSource": "pnp.data", "path": ".", "yarnPnp": "true" }, @@ -40,9 +73,8 @@ { "kind": "file", "source": "package.json", - "locator": "package.json", - "sha256": "65e86ba14f0beebc4573039ac34a58f6dfa0133aa4a9e7f2dcdbb36a4e5c2814" + "locator": "package.json" } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeDeterminismTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeDeterminismTests.cs index b7be1d102..f9c3bc576 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeDeterminismTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeDeterminismTests.cs @@ -125,6 +125,107 @@ public sealed class NodeDeterminismTests : IDisposable #endregion + [Fact] + public async Task LockOnlyProject_EmitsDeclaredOnlyComponents_WithoutRangeAsPurl() + { + WriteFile("package.json", JsonSerializer.Serialize(new + { + name = "root", + version = "1.0.0", + dependencies = new Dictionary + { + ["express"] = "^4.18.2", + ["left-pad"] = "^1.3.0" + } + })); + + WriteFile("package-lock.json", JsonSerializer.Serialize(new + { + name = "root", + version = "1.0.0", + lockfileVersion = 3, + packages = new Dictionary + { + [""] = new + { + name = "root", + version = "1.0.0" + }, + ["node_modules/express"] = new + { + version = "4.18.2", + resolved = "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + integrity = "sha512-deadbeef" + } + } + })); + + var json = await RunAnalyzerAsync(); + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var express = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/express@4.18.2"); + + var expressMeta = express.GetProperty("metadata"); + Assert.Equal("true", expressMeta.GetProperty("declaredOnly").GetString()); + Assert.Equal("package-lock.json", expressMeta.GetProperty("declared.source").GetString()); + Assert.Equal("package-lock.json:node_modules/express", expressMeta.GetProperty("declared.locator").GetString()); + Assert.Equal("^4.18.2", expressMeta.GetProperty("declared.versionSpec").GetString()); + Assert.Equal("4.18.2", expressMeta.GetProperty("declared.resolvedVersion").GetString()); + + var leftPad = components.Single(static element => + element.GetProperty("name").GetString() == "left-pad"); + + Assert.False(leftPad.TryGetProperty("purl", out _)); + Assert.StartsWith("explicit::node::npm::left-pad::sha256:", leftPad.GetProperty("componentKey").GetString(), StringComparison.Ordinal); + + var leftPadMeta = leftPad.GetProperty("metadata"); + Assert.Equal("true", leftPadMeta.GetProperty("declaredOnly").GetString()); + Assert.Equal("package.json", leftPadMeta.GetProperty("declared.source").GetString()); + Assert.Equal("^1.3.0", leftPadMeta.GetProperty("declared.versionSpec").GetString()); + } + + [Fact] + public async Task PnpmLock_IntegrityMissing_EmitsDeclaredOnlyMetadata() + { + WriteFile("package.json", JsonSerializer.Serialize(new + { + name = "root", + version = "1.0.0", + dependencies = new Dictionary + { + ["local-file"] = "file:../local-file-1.0.0.tgz" + } + })); + + var pnpmLock = "lockfileVersion: '6.0'\n" + + "packages:\n" + + " /local-file/1.0.0:\n" + + " resolution: {tarball: file:../local-file-1.0.0.tgz}\n"; + WriteFile("pnpm-lock.yaml", pnpmLock); + + var json = await RunAnalyzerAsync(); + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var localFile = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/local-file@1.0.0"); + + var meta = localFile.GetProperty("metadata"); + Assert.Equal("true", meta.GetProperty("declaredOnly").GetString()); + Assert.Equal("pnpm-lock.yaml", meta.GetProperty("declared.source").GetString()); + Assert.Equal("pnpm-lock.yaml:local-file/1.0.0", meta.GetProperty("declared.locator").GetString()); + Assert.Equal("file:../local-file-1.0.0.tgz", meta.GetProperty("declared.versionSpec").GetString()); + Assert.Equal("1.0.0", meta.GetProperty("declared.resolvedVersion").GetString()); + Assert.Equal("true", meta.GetProperty("lockIntegrityMissing").GetString()); + Assert.Equal("file", meta.GetProperty("lockIntegrityMissingReason").GetString()); + } + #region Entrypoint Ordering [Fact] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeImportWalkerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeImportWalkerTests.cs new file mode 100644 index 000000000..3d2494669 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeImportWalkerTests.cs @@ -0,0 +1,39 @@ +using StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Tests.Node; + +public sealed class NodeImportWalkerTests +{ + [Fact] + public void AnalyzeImports_ParsesEsmImportsAndExports() + { + var content = """ + import foo from "foo"; + export { bar } from "bar"; + export * from "baz"; + """; + + var edges = NodeImportWalker.AnalyzeImports("/repo", "src/index.mjs", content); + + Assert.Contains(edges, e => e.TargetSpecifier == "foo" && e.Kind == "import"); + Assert.Contains(edges, e => e.TargetSpecifier == "bar" && e.Kind == "export-from"); + Assert.Contains(edges, e => e.TargetSpecifier == "baz" && e.Kind == "export-all"); + } + + [Fact] + public void AnalyzeImports_WhenParserFails_UsesTypeScriptRegexFallback() + { + var content = """ + import type { Foo } from "foo"; + export type { Bar } from "bar"; + + interface Thing { name: string } + """; + + var edges = NodeImportWalker.AnalyzeImports("/repo", "src/index.ts", content); + + Assert.Contains(edges, e => e.TargetSpecifier == "foo" && e.Evidence == "ts-regex"); + Assert.Contains(edges, e => e.TargetSpecifier == "bar" && e.Kind == "export-from" && e.Evidence == "ts-regex"); + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLanguageAnalyzerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLanguageAnalyzerTests.cs index 10ecc1de8..4f897a485 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLanguageAnalyzerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLanguageAnalyzerTests.cs @@ -190,6 +190,54 @@ public sealed class NodeLanguageAnalyzerTests cancellationToken); } + [Fact] + public async Task LockOnlyPackageLockEmitsDeclaredOnlyComponentsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "node", "lock-only-package-lock"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] { new NodeLanguageAnalyzer() }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + + [Fact] + public async Task LockOnlyYarnBerryEmitsDeclaredOnlyComponentsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "node", "lock-only-yarn-berry"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] { new NodeLanguageAnalyzer() }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + + [Fact] + public async Task LockOnlyPnpmEmitsDeclaredOnlyComponentsAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "node", "lock-only-pnpm"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] { new NodeLanguageAnalyzer() }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } + [Fact] public async Task PnpmVirtualStoreIsParsedAsync() { @@ -237,4 +285,20 @@ public sealed class NodeLanguageAnalyzerTests analyzers, cancellationToken); } + + [Fact] + public async Task PackageJsonDeclaredOnlyDependenciesUseExplicitKeyAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "node", "declared-only-package-json"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] { new NodeLanguageAnalyzer() }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLockDataTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLockDataTests.cs index 50abc45e2..771b112cd 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLockDataTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeLockDataTests.cs @@ -38,6 +38,34 @@ public sealed class NodeLockDataTests : IDisposable Assert.Empty(result.DeclaredPackages); } + [Fact] + public async Task LoadAsync_EmptyRootPath_DoesNotThrow_WhenCurrentDirectoryHasPackageJson() + { + var originalDirectory = Environment.CurrentDirectory; + var tempDirectory = Path.Combine(Path.GetTempPath(), "node-lock-tests-cwd-" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(tempDirectory); + + try + { + await File.WriteAllTextAsync(Path.Combine(tempDirectory, "package.json"), """ + { + "name": "fixture", + "version": "0.0.0" + } + """); + + Environment.CurrentDirectory = tempDirectory; + + var result = await NodeLockData.LoadAsync(string.Empty, CancellationToken.None); + Assert.Empty(result.DeclaredPackages); + } + finally + { + Environment.CurrentDirectory = originalDirectory; + Directory.Delete(tempDirectory, recursive: true); + } + } + [Fact] public async Task LoadAsync_OnlyPackageJson_CreatesDeclaredOnlyEntries() { @@ -232,7 +260,6 @@ public sealed class NodeLockDataTests : IDisposable [Fact] public async Task LoadPackageLockJson_V3Format_NestedNodeModules() { - // Note: Nested node_modules require explicit name property for correct extraction await File.WriteAllTextAsync(Path.Combine(_tempDir, "package-lock.json"), """ { "lockfileVersion": 3, @@ -241,7 +268,6 @@ public sealed class NodeLockDataTests : IDisposable "version": "1.0.0" }, "node_modules/parent/node_modules/child": { - "name": "child", "version": "2.0.0" } } @@ -253,6 +279,38 @@ public sealed class NodeLockDataTests : IDisposable Assert.Equal(2, result.DeclaredPackages.Count); Assert.Contains(result.DeclaredPackages, e => e.Name == "parent"); Assert.Contains(result.DeclaredPackages, e => e.Name == "child"); + + Assert.True(result.TryGet("node_modules/parent/node_modules/child", "child", "2.0.0", out var entry)); + Assert.NotNull(entry); + Assert.Equal("2.0.0", entry!.Version); + } + + [Fact] + public async Task LoadPackageLockJson_V3Format_NestedNodeModules_ScopedChild() + { + await File.WriteAllTextAsync(Path.Combine(_tempDir, "package-lock.json"), """ + { + "lockfileVersion": 3, + "packages": { + "node_modules/parent": { + "version": "1.0.0" + }, + "node_modules/parent/node_modules/@types/node": { + "version": "20.10.0" + } + } + } + """); + + var result = await NodeLockData.LoadAsync(_tempDir, CancellationToken.None); + + Assert.Equal(2, result.DeclaredPackages.Count); + Assert.Contains(result.DeclaredPackages, e => e.Name == "parent"); + Assert.Contains(result.DeclaredPackages, e => e.Name == "@types/node"); + + Assert.True(result.TryGet("node_modules/parent/node_modules/@types/node", "@types/node", "20.10.0", out var entry)); + Assert.NotNull(entry); + Assert.Equal("20.10.0", entry!.Version); } [Fact] @@ -495,6 +553,35 @@ valid@^2.0.0: Assert.Contains(result.DeclaredPackages, e => e.Name == "valid"); } + [Fact] + public async Task LoadYarnLock_BerryFormat_ParsesResolutionChecksum_AndSkipsMetadata() + { + await File.WriteAllTextAsync(Path.Combine(_tempDir, "yarn.lock"), """ + __metadata: + version: 6 + + "lodash@npm:^4.17.21": + version: 4.17.21 + resolution: "lodash@npm:4.17.21" + checksum: 10c0deadbeef + """); + + var result = await NodeLockData.LoadAsync(_tempDir, CancellationToken.None); + + Assert.Single(result.DeclaredPackages); + Assert.DoesNotContain(result.DeclaredPackages, e => e.Name == "__metadata"); + + var entry = result.DeclaredPackages.Single(); + Assert.Equal("lodash", entry.Name); + Assert.Equal("4.17.21", entry.Version); + Assert.Equal("lodash@npm:4.17.21", entry.Resolved); + Assert.Equal("checksum:10c0deadbeef", entry.Integrity); + + Assert.True(result.TryGet("", "lodash", "4.17.21", out var byVersion)); + Assert.NotNull(byVersion); + Assert.Equal("lodash@npm:^4.17.21", byVersion!.Locator); + } + #endregion #region pnpm-lock.yaml Parsing Tests @@ -557,6 +644,24 @@ valid@^2.0.0: Assert.Equal("4.18.2", result.DeclaredPackages.First().Version); } + [Fact] + public async Task LoadPnpmLock_WhenVersionLineMissing_UsesVersionFromKey() + { + var content = "lockfileVersion: '6.0'\n" + + "packages:\n" + + " /express/4.18.2:\n" + + " resolution: {integrity: sha512-xyz}\n"; + await File.WriteAllTextAsync(Path.Combine(_tempDir, "pnpm-lock.yaml"), content); + + var result = await NodeLockData.LoadAsync(_tempDir, CancellationToken.None); + + Assert.Single(result.DeclaredPackages); + var entry = result.DeclaredPackages.Single(); + Assert.Equal("express", entry.Name); + Assert.Equal("4.18.2", entry.Version); + Assert.Equal("sha512-xyz", entry.Integrity); + } + [Fact] public async Task LoadPnpmLock_ExtractsTarball() { @@ -573,6 +678,43 @@ valid@^2.0.0: Assert.Contains("lodash-4.17.21.tgz", result.DeclaredPackages.First().Resolved); } + [Fact] + public async Task LoadPnpmLock_IntegrityMissingReason_File() + { + var content = "lockfileVersion: '6.0'\n" + + "packages:\n" + + " /local-file/1.0.0:\n" + + " resolution: {tarball: file:../local-file-1.0.0.tgz}\n"; + await File.WriteAllTextAsync(Path.Combine(_tempDir, "pnpm-lock.yaml"), content); + + var result = await NodeLockData.LoadAsync(_tempDir, CancellationToken.None); + + Assert.Single(result.DeclaredPackages); + var entry = result.DeclaredPackages.Single(); + Assert.Equal("local-file", entry.Name); + Assert.True(entry.IntegrityMissing); + Assert.Equal("file", entry.IntegrityMissingReason); + Assert.StartsWith("file:", entry.Resolved, StringComparison.Ordinal); + } + + [Fact] + public async Task LoadPnpmLock_SnapshotsSection_IsParsed() + { + var content = "lockfileVersion: '9.0'\n" + + "snapshots:\n" + + " /snap-only/1.0.0:\n" + + " resolution: {integrity: sha512-snap}\n"; + await File.WriteAllTextAsync(Path.Combine(_tempDir, "pnpm-lock.yaml"), content); + + var result = await NodeLockData.LoadAsync(_tempDir, CancellationToken.None); + + Assert.Single(result.DeclaredPackages); + var entry = result.DeclaredPackages.Single(); + Assert.Equal("snap-only", entry.Name); + Assert.Equal("1.0.0", entry.Version); + Assert.Equal("sha512-snap", entry.Integrity); + } + [Fact] public async Task LoadPnpmLock_SeparateIntegrityLine() { @@ -590,7 +732,7 @@ valid@^2.0.0: } [Fact] - public async Task LoadPnpmLock_SkipsPackagesWithoutIntegrity() + public async Task LoadPnpmLock_PackagesWithoutIntegrity_AreKeptAndMarked() { var content = "lockfileVersion: '6.0'\n" + "packages:\n" + @@ -603,8 +745,19 @@ valid@^2.0.0: var result = await NodeLockData.LoadAsync(_tempDir, CancellationToken.None); - Assert.Single(result.DeclaredPackages); - Assert.Equal("has-integrity", result.DeclaredPackages.First().Name); + Assert.Equal(2, result.DeclaredPackages.Count); + + var noIntegrity = result.DeclaredPackages.Single(e => e.Name == "no-integrity"); + Assert.Equal("1.0.0", noIntegrity.Version); + Assert.Null(noIntegrity.Integrity); + Assert.True(noIntegrity.IntegrityMissing); + Assert.Equal("missing", noIntegrity.IntegrityMissingReason); + + var hasIntegrity = result.DeclaredPackages.Single(e => e.Name == "has-integrity"); + Assert.Equal("2.0.0", hasIntegrity.Version); + Assert.Equal("sha512-valid", hasIntegrity.Integrity); + Assert.False(hasIntegrity.IntegrityMissing); + Assert.Null(hasIntegrity.IntegrityMissingReason); } [Fact] @@ -863,6 +1016,12 @@ valid@^2.0.0: Assert.True(result.TryGet("", "lodash", out var byNameEntry)); Assert.Equal("4.0.0", byNameEntry!.Version); + Assert.True(result.TryGet("", "lodash", "4.17.21", out var byVersionEntry)); + Assert.Equal("4.17.21", byVersionEntry!.Version); + + Assert.True(result.TryGet("", "lodash", "4.0.0", out var byVersionEntry2)); + Assert.Equal("4.0.0", byVersionEntry2!.Version); + // For TryGet lookups by path, package-lock.json entry is found Assert.True(result.TryGet("node_modules/lodash", "", out var byPathEntry)); Assert.Equal("4.17.21", byPathEntry!.Version); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodePackageCollectorTraversalTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodePackageCollectorTraversalTests.cs index 60708a2ba..66edfe3cf 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodePackageCollectorTraversalTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodePackageCollectorTraversalTests.cs @@ -611,6 +611,177 @@ public sealed class NodePackageCollectorTraversalTests : IDisposable #endregion + #region Lock Metadata Attachment + + [Fact] + public async Task Traversal_NestedNodeModules_AttachesCorrectLockMetadata() + { + WritePackageJson(_tempDir, "root-app", "1.0.0", isPrivate: true); + + var nodeModules = Path.Combine(_tempDir, "node_modules"); + WritePackageJson(Path.Combine(nodeModules, "parent"), "parent", "1.0.0"); + WritePackageJson(Path.Combine(nodeModules, "parent", "node_modules", "child"), "child", "2.0.0"); + + await File.WriteAllTextAsync(Path.Combine(_tempDir, "package-lock.json"), """ + { + "lockfileVersion": 3, + "packages": { + "node_modules/parent": { + "version": "1.0.0", + "integrity": "sha512-parent" + }, + "node_modules/parent/node_modules/child": { + "version": "2.0.0", + "resolved": "https://example.com/child.tgz", + "integrity": "sha512-child" + } + } + } + """); + + var json = await RunAnalyzerAsync(); + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var child = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/child@2.0.0"); + + var metadata = child.GetProperty("metadata"); + Assert.Equal("sha512-child", metadata.GetProperty("integrity").GetString()); + Assert.Equal("https://example.com/child.tgz", metadata.GetProperty("resolved").GetString()); + Assert.Equal("package-lock.json", metadata.GetProperty("lockSource").GetString()); + Assert.Equal("package-lock.json:node_modules/parent/node_modules/child", metadata.GetProperty("lockLocator").GetString()); + } + + #endregion + + #region Workspace Scope Attribution + + [Fact] + public async Task Traversal_WorkspaceDependencyScopes_AreDerivedFromWorkspaceManifest() + { + File.WriteAllText(Path.Combine(_tempDir, "package.json"), """ + { + "name": "root-app", + "version": "1.0.0", + "private": true, + "workspaces": ["packages/*"] + } + """); + + var workspaceDir = Path.Combine(_tempDir, "packages", "app"); + Directory.CreateDirectory(workspaceDir); + File.WriteAllText(Path.Combine(workspaceDir, "package.json"), """ + { + "name": "app", + "version": "1.0.0", + "dependencies": { + "prod-dep": "^1.0.0" + }, + "devDependencies": { + "dev-dep": "^1.0.0" + }, + "optionalDependencies": { + "opt-dep": "^1.0.0" + } + } + """); + + var nodeModules = Path.Combine(workspaceDir, "node_modules"); + WritePackageJson(Path.Combine(nodeModules, "prod-dep"), "prod-dep", "1.0.0"); + WritePackageJson(Path.Combine(nodeModules, "dev-dep"), "dev-dep", "1.0.0"); + WritePackageJson(Path.Combine(nodeModules, "opt-dep"), "opt-dep", "1.0.0"); + + var json = await RunAnalyzerAsync(); + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var prod = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/prod-dep@1.0.0"); + Assert.Equal("production", prod.GetProperty("metadata").GetProperty("scope").GetString()); + Assert.Equal("production", prod.GetProperty("metadata").GetProperty("riskLevel").GetString()); + + var dev = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/dev-dep@1.0.0"); + Assert.Equal("development", dev.GetProperty("metadata").GetProperty("scope").GetString()); + Assert.Equal("development", dev.GetProperty("metadata").GetProperty("riskLevel").GetString()); + + var opt = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/opt-dep@1.0.0"); + Assert.Equal("optional", opt.GetProperty("metadata").GetProperty("scope").GetString()); + Assert.Equal("optional", opt.GetProperty("metadata").GetProperty("riskLevel").GetString()); + Assert.Equal("true", opt.GetProperty("metadata").GetProperty("optional").GetString()); + } + + #endregion + + #region Import Scanning Bounds + + [Fact] + public async Task Traversal_ImportScan_SkipsNodeModulesPackagesByDefault() + { + WritePackageJson(_tempDir, "root-app", "1.0.0", isPrivate: true); + Directory.CreateDirectory(Path.Combine(_tempDir, "src")); + File.WriteAllText(Path.Combine(_tempDir, "src", "index.js"), "import foo from \"foo\";\n"); + + var fooDir = Path.Combine(_tempDir, "node_modules", "foo"); + WritePackageJson(fooDir, "foo", "1.0.0"); + File.WriteAllText(Path.Combine(fooDir, "index.js"), "import bar from \"bar\";\n"); + + var json = await RunAnalyzerAsync(); + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var root = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/root-app@1.0.0"); + Assert.Contains(root.GetProperty("evidence").EnumerateArray(), e => e.GetProperty("source").GetString() == "node.import"); + + var foo = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/foo@1.0.0"); + Assert.DoesNotContain(foo.GetProperty("evidence").EnumerateArray(), e => e.GetProperty("source").GetString() == "node.import"); + } + + [Fact] + public async Task Traversal_ImportScan_WhenCapped_EmitsImportScanSkippedMetadata() + { + WritePackageJson(_tempDir, "root-app", "1.0.0", isPrivate: true); + var srcDir = Path.Combine(_tempDir, "src"); + Directory.CreateDirectory(srcDir); + + for (var i = 0; i < 510; i++) + { + File.WriteAllText(Path.Combine(srcDir, $"file-{i:D4}.js"), ";\n"); + } + + var json = await RunAnalyzerAsync(); + using var document = JsonDocument.Parse(json); + var components = document.RootElement.EnumerateArray().ToArray(); + + var root = components.Single(static element => + element.TryGetProperty("purl", out var purl) + && purl.ValueKind == JsonValueKind.String + && purl.GetString() == "pkg:npm/root-app@1.0.0"); + + var metadata = root.GetProperty("metadata"); + Assert.Equal("true", metadata.GetProperty("importScanSkipped").GetString()); + Assert.Equal("500", metadata.GetProperty("importScan.filesScanned").GetString()); + Assert.True(metadata.TryGetProperty("importScan.bytesScanned", out _)); + } + + #endregion + #region Deeply Nested Packages [Fact] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeWorkspaceIndexTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeWorkspaceIndexTests.cs new file mode 100644 index 000000000..c5e527005 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/Node/NodeWorkspaceIndexTests.cs @@ -0,0 +1,102 @@ +using StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Tests.Node; + +public sealed class NodeWorkspaceIndexTests : IDisposable +{ + private readonly string _tempDir; + + public NodeWorkspaceIndexTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), "node-workspace-index-tests-" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(_tempDir); + } + + public void Dispose() + { + if (Directory.Exists(_tempDir)) + { + try + { + Directory.Delete(_tempDir, recursive: true); + } + catch + { + // Ignore cleanup failures in tests + } + } + } + + private void WriteFile(string relativePath, string content) + { + var fullPath = Path.Combine(_tempDir, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + File.WriteAllText(fullPath, content); + } + + [Fact] + public void Create_NoPackageJson_ReturnsEmpty() + { + var index = NodeWorkspaceIndex.Create(_tempDir); + + Assert.Empty(index.GetMembers()); + } + + [Fact] + public void Create_PackagesStar_ExpandsImmediateChildren_Deterministically() + { + WriteFile("package.json", """ + { + "workspaces": ["packages/*", "./tools/*"] + } + """); + + WriteFile("packages/a/package.json", """{ "name": "a", "version": "1.0.0" }"""); + WriteFile("packages/b/package.json", """{ "name": "b", "version": "1.0.0" }"""); + Directory.CreateDirectory(Path.Combine(_tempDir, "packages", "empty")); + + WriteFile("tools/t1/package.json", """{ "name": "t1", "version": "1.0.0" }"""); + + var index = NodeWorkspaceIndex.Create(_tempDir); + + Assert.Equal(new[] { "packages/a", "packages/b", "tools/t1" }, index.GetMembers().ToArray()); + } + + [Fact] + public void Create_DoubleStar_ExpandsNestedMembers_AndSkipsNodeModules() + { + WriteFile("package.json", """ + { + "workspaces": ["apps/**"] + } + """); + + WriteFile("apps/app2/package.json", """{ "name": "app2", "version": "1.0.0" }"""); + WriteFile("apps/team/app1/package.json", """{ "name": "app1", "version": "1.0.0" }"""); + WriteFile("apps/team/node_modules/evil/package.json", """{ "name": "evil", "version": "1.0.0" }"""); + + var index = NodeWorkspaceIndex.Create(_tempDir); + + Assert.Equal(new[] { "apps/app2", "apps/team/app1" }, index.GetMembers().ToArray()); + Assert.False(index.TryGetMember("apps/team/node_modules/evil", out _)); + } + + [Fact] + public void Create_PackagesDoubleStar_DoesNotIncludeNodeModulesTree() + { + WriteFile("package.json", """ + { + "workspaces": ["packages/**"] + } + """); + + WriteFile("packages/ok/package.json", """{ "name": "ok", "version": "1.0.0" }"""); + WriteFile("packages/node_modules/evil/package.json", """{ "name": "evil", "version": "1.0.0" }"""); + + var index = NodeWorkspaceIndex.Create(_tempDir); + + Assert.Single(index.GetMembers()); + Assert.Equal("packages/ok", index.GetMembers().Single()); + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests.csproj index 8688b1faa..4bd24d696 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests/StellaOps.Scanner.Analyzers.Lang.Node.Tests.csproj @@ -17,12 +17,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests.csproj index 4ae56b52a..814bab793 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests/StellaOps.Scanner.Analyzers.Lang.Php.Tests.csproj @@ -14,12 +14,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Entrypoints/PythonEntrypointDiscoveryTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Entrypoints/PythonEntrypointDiscoveryTests.cs index 83729d0b8..e35a38581 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Entrypoints/PythonEntrypointDiscoveryTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Entrypoints/PythonEntrypointDiscoveryTests.cs @@ -1,5 +1,6 @@ using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Entrypoints; using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem; +using System.IO.Compression; namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.Entrypoints; @@ -86,6 +87,40 @@ mygui = mypackage.gui:start } } + [Fact] + public async Task DiscoverAsync_FindsZipappMain() + { + var cancellationToken = TestContext.Current.CancellationToken; + var tempPath = CreateTemporaryWorkspace(); + try + { + var zipappPath = Path.Combine(tempPath, "app.pyz"); + using (var stream = File.Create(zipappPath)) + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false)) + { + var entry = archive.CreateEntry("__main__.py"); + await using var entryStream = entry.Open(); + await using var writer = new StreamWriter(entryStream); + await writer.WriteAsync("print('hello')".AsMemory(), cancellationToken); + } + + var vfs = PythonVirtualFileSystem.CreateBuilder() + .AddZipapp(zipappPath) + .Build(); + + var discovery = new PythonEntrypointDiscovery(vfs, tempPath); + await discovery.DiscoverAsync(cancellationToken); + + Assert.Contains(discovery.Entrypoints, e => + e.Kind == PythonEntrypointKind.ZipappMain && + e.Name == "__main__"); + } + finally + { + Directory.Delete(tempPath, recursive: true); + } + } + [Fact] public async Task DiscoverAsync_FindsDjangoManage() { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info/METADATA b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info/METADATA new file mode 100644 index 000000000..f97d996d1 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info/METADATA @@ -0,0 +1,4 @@ +Metadata-Version: 2.1 +Name: hiddenpkg +Version: 0.1.0 +Summary: Hidden layer package diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg/__init__.py b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg/__init__.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/.layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json index 420f78446..f0bb88ea6 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/expected.json @@ -1,4 +1,38 @@ [ + { + "analyzerId": "python", + "componentKey": "purl::pkg:pypi/hiddenpkg@0.1.0", + "purl": "pkg:pypi/hiddenpkg@0.1.0", + "name": "hiddenpkg", + "version": "0.1.0", + "type": "pypi", + "usedByEntrypoint": false, + "metadata": { + "distInfoPath": ".layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info", + "name": "hiddenpkg", + "normalizedName": "hiddenpkg", + "pkg.confidence": "High", + "pkg.kind": "Wheel", + "pkg.location": ".layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info", + "provenance": "dist-info", + "record.hashMismatches": "0", + "record.hashedEntries": "0", + "record.ioErrors": "0", + "record.missingFiles": "0", + "record.totalEntries": "0", + "runtime.libPaths.count": "2", + "runtime.versions": "3.11", + "summary": "Hidden layer package", + "version": "0.1.0" + }, + "evidence": [ + { + "kind": "file", + "source": "METADATA", + "locator": ".layers/layer0/usr/lib/python3.11/site-packages/hiddenpkg-0.1.0.dist-info/METADATA" + } + ] + }, { "analyzerId": "python", "componentKey": "purl::pkg:pypi/layered@2.0", @@ -6,37 +40,38 @@ "name": "layered", "version": "2.0", "type": "pypi", - "usedByEntrypoint": true, + "usedByEntrypoint": false, "metadata": { "author": "Layered Maintainer", "authorEmail": "maintainer@example.com", - "classifier[0]": "Programming Language :: Python :: 3", - "classifiers": "Programming Language :: Python :: 3", - "distInfoPath": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info", + "classifier[0]": "License :: OSI Approved :: Apache Software License", + "classifiers": "License :: OSI Approved :: Apache Software License", + "distInfoPath": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info", "editable": "true", - "entryPoints.console_scripts": "layered-cli=layered.cli:main", "entryPoints.layered.hooks": "register=layered.plugins:register", "installer": "pip", - "license": "Apache-2.0", "license.classifier[0]": "License :: OSI Approved :: Apache Software License", "license.file[0]": "layer2/usr/lib/python3.11/site-packages/LICENSE", "licenseExpression": "Apache-2.0", "name": "layered", "normalizedName": "layered", + "pkg.confidence": "Definitive", + "pkg.kind": "Wheel", + "pkg.location": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info", "projectUrl": "Documentation, https://example.com/layered/docs", "provenance": "dist-info", "record.hashMismatches": "0", - "record.hashedEntries": "7", + "record.hashedEntries": "8", "record.ioErrors": "0", - "record.missingFiles": "1", + "record.missingFiles": "0", "record.totalEntries": "9", - "requiresDist": "requests", - "requiresPython": "\u003E=3.9", + "runtime.libPaths.count": "2", + "runtime.versions": "3.11", "sourceCommit": "abc123", "sourceSubdirectory": "src/layered", "sourceUrl": "https://git.example.com/layered", "sourceVcs": "git", - "summary": "Base layer metadata", + "summary": "Overlay metadata adding direct URL information", "version": "2.0", "wheel.generator": "pip 24.0", "wheel.rootIsPurelib": "true", @@ -44,57 +79,26 @@ "wheel.version": "1.0" }, "evidence": [ - { - "kind": "derived", - "source": "RECORD", - "locator": "layer1/usr/bin/layered-cli", - "value": "missing" - }, - { - "kind": "file", - "source": "INSTALLER", - "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER" - }, { "kind": "file", "source": "INSTALLER", "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/INSTALLER" }, - { - "kind": "file", - "source": "METADATA", - "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA" - }, { "kind": "file", "source": "METADATA", "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/METADATA" }, - { - "kind": "file", - "source": "RECORD", - "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD" - }, { "kind": "file", "source": "RECORD", "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/RECORD" }, - { - "kind": "file", - "source": "WHEEL", - "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL" - }, { "kind": "file", "source": "WHEEL", "locator": "layer2/usr/lib/python3.11/site-packages/layered-2.0.dist-info/WHEEL" }, - { - "kind": "file", - "source": "entry_points.txt", - "locator": "layer1/usr/lib/python3.11/site-packages/layered-2.0.dist-info/entry_points.txt" - }, { "kind": "file", "source": "entry_points.txt", @@ -112,5 +116,39 @@ "value": "https://git.example.com/layered" } ] + }, + { + "analyzerId": "python", + "componentKey": "purl::pkg:pypi/layerspkg@0.2.0", + "purl": "pkg:pypi/layerspkg@0.2.0", + "name": "layerspkg", + "version": "0.2.0", + "type": "pypi", + "usedByEntrypoint": false, + "metadata": { + "distInfoPath": "layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info", + "name": "layerspkg", + "normalizedName": "layerspkg", + "pkg.confidence": "High", + "pkg.kind": "Wheel", + "pkg.location": "layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info", + "provenance": "dist-info", + "record.hashMismatches": "0", + "record.hashedEntries": "0", + "record.ioErrors": "0", + "record.missingFiles": "0", + "record.totalEntries": "0", + "runtime.libPaths.count": "2", + "runtime.versions": "3.11", + "summary": "Layers/ root package", + "version": "0.2.0" + }, + "evidence": [ + { + "kind": "file", + "source": "METADATA", + "locator": "layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info/METADATA" + } + ] } ] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info/METADATA b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info/METADATA new file mode 100644 index 000000000..928afc8b4 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg-0.2.0.dist-info/METADATA @@ -0,0 +1,4 @@ +Metadata-Version: 2.1 +Name: layerspkg +Version: 0.2.0 +Summary: Layers/ root package diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg/__init__.py b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg/__init__.py new file mode 100644 index 000000000..d3ec452c3 --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/layered-editable/layers/layer3/usr/lib/python3.11/site-packages/layerspkg/__init__.py @@ -0,0 +1 @@ +__version__ = "0.2.0" diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json index 9a497a0a4..cc9e5c36d 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pip-cache/expected.json @@ -20,6 +20,9 @@ "license.file[0]": "LICENSE", "name": "Cache-Pkg", "normalizedName": "cache-pkg", + "pkg.confidence": "Definitive", + "pkg.kind": "Wheel", + "pkg.location": "lib/python3.11/site-packages/cache_pkg-1.2.3.dist-info", "projectUrl": "Source, https://example.com/cache-pkg", "provenance": "dist-info", "record.hashMismatches": "1", @@ -86,4 +89,4 @@ } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json index 6a356508d..39f7db8a9 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/simple-venv/expected.json @@ -22,6 +22,9 @@ "license.classifier[0]": "License :: OSI Approved :: Apache Software License", "name": "simple", "normalizedName": "simple", + "pkg.confidence": "Definitive", + "pkg.kind": "Wheel", + "pkg.location": "lib/python3.11/site-packages/simple-1.0.0.dist-info", "projectUrl": "Source, https://example.com/simple/src", "provenance": "dist-info", "record.hashMismatches": "0", @@ -84,4 +87,4 @@ } ] } -] +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs index 6b8ae22e4..786b4c2d7 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs @@ -1,3 +1,4 @@ +using System.IO.Compression; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -125,6 +126,137 @@ public sealed class PythonLanguageAnalyzerTests } } + [Fact] + public async Task EditableRequirementsUseExplicitKeyWithoutHostPathLeakAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = CreateTemporaryWorkspace(); + + try + { + var editableDir = Path.Combine(fixturePath, "editable-src"); + Directory.CreateDirectory(editableDir); + + var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); + await File.WriteAllTextAsync(requirementsPath, $"--editable {editableDir}{Environment.NewLine}", cancellationToken); + + var analyzers = new ILanguageAnalyzer[] + { + new PythonLanguageAnalyzer() + }; + + var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( + fixturePath, + analyzers, + cancellationToken); + + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + foreach (var component in root.EnumerateArray()) + { + if (component.TryGetProperty("purl", out var purlElement) && purlElement.ValueKind == JsonValueKind.String) + { + Assert.DoesNotContain("@editable", purlElement.GetString(), StringComparison.OrdinalIgnoreCase); + } + } + + var editableComponent = root.EnumerateArray().Single(static component => + component.TryGetProperty("name", out var nameElement) + && string.Equals("editable-src", nameElement.GetString(), StringComparison.OrdinalIgnoreCase)); + + Assert.True(!editableComponent.TryGetProperty("purl", out var purlValue) || purlValue.ValueKind == JsonValueKind.Null); + + var componentKey = editableComponent.GetProperty("componentKey").GetString(); + Assert.StartsWith("explicit::python::pypi::editable-src::sha256:", componentKey, StringComparison.Ordinal); + + var metadata = editableComponent.GetProperty("metadata"); + Assert.Equal("true", metadata.GetProperty("declaredOnly").GetString()); + Assert.Equal("editable", metadata.GetProperty("declared.sourceType").GetString()); + Assert.Equal("requirements.txt", metadata.GetProperty("declared.source").GetString()); + Assert.Equal("requirements.txt", metadata.GetProperty("declared.locator").GetString()); + + var editableSpec = metadata.GetProperty("lockEditablePath").GetString(); + Assert.Equal("editable-src", editableSpec); + Assert.DoesNotContain(fixturePath, editableSpec, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain(":", editableSpec, StringComparison.Ordinal); + } + finally + { + Directory.Delete(fixturePath, recursive: true); + } + } + + [Fact] + public async Task WheelArchiveDistInfo_IsVerifiedFromRecordAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = CreateTemporaryWorkspace(); + try + { + var distDir = Path.Combine(fixturePath, "dist"); + Directory.CreateDirectory(distDir); + + var wheelPath = Path.Combine(distDir, "archivepkg-1.0.0-py3-none-any.whl"); + + var initBytes = Encoding.UTF8.GetBytes("__version__ = \"1.0.0\"\n"); + var metadataBytes = Encoding.UTF8.GetBytes( + $"Metadata-Version: 2.1\nName: archivepkg\nVersion: 1.0.0\n{Environment.NewLine}"); + var wheelBytes = Encoding.UTF8.GetBytes( + "Wheel-Version: 1.0\nGenerator: test\nRoot-Is-Purelib: true\nTag: py3-none-any\n"); + + var recordContent = new StringBuilder() + .AppendLine($"archivepkg/__init__.py,sha256={ComputeSha256Base64(initBytes)},{initBytes.Length}") + .AppendLine($"archivepkg-1.0.0.dist-info/METADATA,sha256={ComputeSha256Base64(metadataBytes)},{metadataBytes.Length}") + .AppendLine($"archivepkg-1.0.0.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelBytes)},{wheelBytes.Length}") + .AppendLine("archivepkg-1.0.0.dist-info/RECORD,,") + .ToString(); + var recordBytes = Encoding.UTF8.GetBytes(recordContent); + + using (var stream = File.Create(wheelPath)) + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false)) + { + WriteEntry(archive, "archivepkg/__init__.py", initBytes); + WriteEntry(archive, "archivepkg-1.0.0.dist-info/METADATA", metadataBytes); + WriteEntry(archive, "archivepkg-1.0.0.dist-info/WHEEL", wheelBytes); + WriteEntry(archive, "archivepkg-1.0.0.dist-info/RECORD", recordBytes); + } + + var analyzers = new ILanguageAnalyzer[] + { + new PythonLanguageAnalyzer() + }; + + var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( + fixturePath, + analyzers, + cancellationToken); + + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + Assert.True(ComponentHasMetadata(root, "archivepkg", "record.totalEntries", "4")); + Assert.True(ComponentHasMetadata(root, "archivepkg", "record.hashedEntries", "3")); + Assert.True(ComponentHasMetadata(root, "archivepkg", "record.missingFiles", "0")); + Assert.True(ComponentHasMetadata(root, "archivepkg", "record.hashMismatches", "0")); + Assert.True(ComponentHasMetadata(root, "archivepkg", "record.ioErrors", "0")); + } + finally + { + Directory.Delete(fixturePath, recursive: true); + } + + static void WriteEntry(ZipArchive archive, string entryName, byte[] content) + { + var entry = archive.CreateEntry(entryName); + using var entryStream = entry.Open(); + entryStream.Write(content, 0, content.Length); + } + + static string ComputeSha256Base64(byte[] content) + => Convert.ToBase64String(SHA256.HashData(content)); + } + private static async Task CreatePythonPackageAsync(string root, string name, string version, CancellationToken cancellationToken) { var sitePackages = Path.Combine(root, "lib", "python3.11", "site-packages"); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests.csproj index b26582d41..077d00eed 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests.csproj @@ -14,12 +14,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/VirtualFileSystem/PythonVirtualFileSystemTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/VirtualFileSystem/PythonVirtualFileSystemTests.cs index 93689dbc4..771a2172f 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/VirtualFileSystem/PythonVirtualFileSystemTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/VirtualFileSystem/PythonVirtualFileSystemTests.cs @@ -100,11 +100,12 @@ public sealed class PythonVirtualFileSystemTests .Build(); Assert.Equal(3, vfs.FileCount); - Assert.True(vfs.FileExists("mypackage/__init__.py")); - Assert.True(vfs.FileExists("mypackage/core.py")); - Assert.True(vfs.FileExists("mypackage-1.0.0.dist-info/METADATA")); + var wheelRoot = $"archives/wheel/{Path.GetFileName(wheelPath)}"; + Assert.True(vfs.FileExists($"{wheelRoot}/mypackage/__init__.py")); + Assert.True(vfs.FileExists($"{wheelRoot}/mypackage/core.py")); + Assert.True(vfs.FileExists($"{wheelRoot}/mypackage-1.0.0.dist-info/METADATA")); - Assert.True(vfs.TryGetFile("mypackage/__init__.py", out var file)); + Assert.True(vfs.TryGetFile($"{wheelRoot}/mypackage/__init__.py", out var file)); Assert.Equal(PythonFileSource.Wheel, file!.Source); Assert.Equal(wheelPath, file.ArchivePath); } @@ -141,9 +142,10 @@ public sealed class PythonVirtualFileSystemTests .Build(); Assert.Equal(1, vfs.FileCount); - Assert.True(vfs.FileExists("__main__.py")); + var zipappRoot = $"archives/zipapp/{Path.GetFileName(zipappPath)}"; + Assert.True(vfs.FileExists($"{zipappRoot}/__main__.py")); - Assert.True(vfs.TryGetFile("__main__.py", out var file)); + Assert.True(vfs.TryGetFile($"{zipappRoot}/__main__.py", out var file)); Assert.Equal(PythonFileSource.Zipapp, file!.Source); } finally @@ -252,7 +254,7 @@ public sealed class PythonVirtualFileSystemTests Assert.Single(sitePackagesFiles); Assert.Single(wheelFiles); Assert.Equal("installed/__init__.py", sitePackagesFiles[0].VirtualPath); - Assert.Equal("wheel/__init__.py", wheelFiles[0].VirtualPath); + Assert.Equal($"archives/wheel/{Path.GetFileName(wheelPath)}/wheel/__init__.py", wheelFiles[0].VirtualPath); } finally { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/cli-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/cli-app/expected.json index a59e45c75..059652297 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/cli-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/cli-app/expected.json @@ -12,6 +12,7 @@ "ruby.observation.capability.schedulers": "0", "ruby.observation.capability.serialization": "false", "ruby.observation.dependency_edges": "6", + "ruby.observation.entrypoints": "0", "ruby.observation.packages": "9", "ruby.observation.ruby_version": "3.2.0", "ruby.observation.runtime_edges": "0" diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/complex-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/complex-app/expected.json index 736867cf4..0d2b4be5e 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/complex-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/complex-app/expected.json @@ -7,12 +7,13 @@ "usedByEntrypoint": false, "metadata": { "ruby.observation.bundler_version": "2.5.3", - "ruby.observation.capability.exec": "false", + "ruby.observation.capability.exec": "true", "ruby.observation.capability.net": "true", "ruby.observation.capability.scheduler_list": "clockwork;sidekiq", "ruby.observation.capability.schedulers": "2", "ruby.observation.capability.serialization": "false", "ruby.observation.dependency_edges": "4", + "ruby.observation.entrypoints": "1", "ruby.observation.packages": "6", "ruby.observation.runtime_edges": "5" }, @@ -21,8 +22,8 @@ "kind": "derived", "source": "ruby.observation", "locator": "document", - "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022clockwork\u0022,\u0022version\u0022:\u00223.0.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022ops\u0022]},{\u0022name\u0022:\u0022pagy\u0022,\u0022version\u0022:\u00226.5.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022web\u0022]},{\u0022name\u0022:\u0022pry\u0022,\u0022version\u0022:\u00220.14.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022tools\u0022]},{\u0022name\u0022:\u0022rack\u0022,\u0022version\u0022:\u00223.1.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022version\u0022:\u00227.2.1\u0022,\u0022source\u0022:\u0022vendor\u0022,\u0022declaredOnly\u0022:false,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022artifact\u0022:\u0022vendor/custom-bundle/cache/sidekiq-7.2.1.gem\u0022,\u0022groups\u0022:[\u0022jobs\u0022]},{\u0022name\u0022:\u0022sinatra\u0022,\u0022version\u0022:\u00223.1.0\u0022,\u0022source\u0022:\u0022vendor-cache\u0022,\u0022declaredOnly\u0022:false,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022artifact\u0022:\u0022vendor/cache/sinatra-3.1.0.gem\u0022,\u0022groups\u0022:[\u0022web\u0022]}],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022config/environment.rb\u0022,\u0022type\u0022:\u0022script\u0022,\u0022requiredGems\u0022:[\u0022pagy\u0022]}],\u0022dependencyEdges\u0022:[{\u0022from\u0022:\u0022pkg:gem/pry@0.14.2\u0022,\u0022to\u0022:\u0022coderay\u0022,\u0022constraint\u0022:\u0022~\\u003E 1.1\u0022},{\u0022from\u0022:\u0022pkg:gem/pry@0.14.2\u0022,\u0022to\u0022:\u0022method_source\u0022,\u0022constraint\u0022:\u0022~\\u003E 1.0\u0022},{\u0022from\u0022:\u0022pkg:gem/sidekiq@7.2.1\u0022,\u0022to\u0022:\u0022rack\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.0\u0022},{\u0022from\u0022:\u0022pkg:gem/sinatra@3.1.0\u0022,\u0022to\u0022:\u0022rack\u0022,\u0022constraint\u0022:\u0022~\\u003E 3.0\u0022}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022clockwork\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022scripts/worker.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022pagy\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022,\u0022config/environment.rb\u0022],\u0022entrypoints\u0022:[\u0022config/environment.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rack\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022sidekiq\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022scripts/worker.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022sinatra\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022jobs\u0022:[{\u0022name\u0022:\u0022clockwork\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022clockwork\u0022},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022sidekiq\u0022}],\u0022environment\u0022:{\u0022bundlerVersion\u0022:\u00222.5.3\u0022,\u0022bundlePaths\u0022:[\u0022/mnt/e/dev/git.stella-ops.org/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/bin/Debug/net10.0/Fixtures/lang/ruby/complex-app/vendor/custom-bundle\u0022],\u0022gemfiles\u0022:[\u0022/mnt/e/dev/git.stella-ops.org/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/bin/Debug/net10.0/Fixtures/lang/ruby/complex-app/Gemfile\u0022],\u0022lockfiles\u0022:[\u0022Gemfile.lock\u0022],\u0022frameworks\u0022:[\u0022clockwork\u0022,\u0022sidekiq\u0022]},\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[\u0022clockwork\u0022,\u0022sidekiq\u0022]},\u0022bundledWith\u0022:\u00222.5.3\u0022}", - "sha256": "sha256:bd15160e034ea5adf0a8384dc9ee18557f695b0952d4fb17214f1bd1381ad22a" + "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022clockwork\u0022,\u0022version\u0022:\u00223.0.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022ops\u0022]},{\u0022name\u0022:\u0022pagy\u0022,\u0022version\u0022:\u00226.5.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022web\u0022]},{\u0022name\u0022:\u0022pry\u0022,\u0022version\u0022:\u00220.14.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022tools\u0022]},{\u0022name\u0022:\u0022rack\u0022,\u0022version\u0022:\u00223.1.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022version\u0022:\u00227.2.1\u0022,\u0022source\u0022:\u0022vendor\u0022,\u0022declaredOnly\u0022:false,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022artifact\u0022:\u0022vendor/custom-bundle/cache/sidekiq-7.2.1.gem\u0022,\u0022groups\u0022:[\u0022jobs\u0022]},{\u0022name\u0022:\u0022sinatra\u0022,\u0022version\u0022:\u00223.1.0\u0022,\u0022source\u0022:\u0022vendor-cache\u0022,\u0022declaredOnly\u0022:false,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022artifact\u0022:\u0022vendor/cache/sinatra-3.1.0.gem\u0022,\u0022groups\u0022:[\u0022web\u0022]}],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022config/environment.rb\u0022,\u0022type\u0022:\u0022script\u0022,\u0022requiredGems\u0022:[\u0022pagy\u0022]}],\u0022dependencyEdges\u0022:[{\u0022from\u0022:\u0022pkg:gem/pry@0.14.2\u0022,\u0022to\u0022:\u0022coderay\u0022,\u0022constraint\u0022:\u0022~\\u003E 1.1\u0022},{\u0022from\u0022:\u0022pkg:gem/pry@0.14.2\u0022,\u0022to\u0022:\u0022method_source\u0022,\u0022constraint\u0022:\u0022~\\u003E 1.0\u0022},{\u0022from\u0022:\u0022pkg:gem/sidekiq@7.2.1\u0022,\u0022to\u0022:\u0022rack\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.0\u0022},{\u0022from\u0022:\u0022pkg:gem/sinatra@3.1.0\u0022,\u0022to\u0022:\u0022rack\u0022,\u0022constraint\u0022:\u0022~\\u003E 3.0\u0022}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022clockwork\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022scripts/worker.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022pagy\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022,\u0022config/environment.rb\u0022],\u0022entrypoints\u0022:[\u0022config/environment.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rack\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022sidekiq\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022scripts/worker.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022sinatra\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022jobs\u0022:[{\u0022name\u0022:\u0022clockwork\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022clockwork\u0022},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022sidekiq\u0022}],\u0022environment\u0022:{\u0022bundlerVersion\u0022:\u00222.5.3\u0022,\u0022bundlePaths\u0022:[\u0022vendor/custom-bundle\u0022],\u0022gemfiles\u0022:[\u0022Gemfile\u0022],\u0022lockfiles\u0022:[\u0022Gemfile.lock\u0022],\u0022frameworks\u0022:[\u0022clockwork\u0022,\u0022sidekiq\u0022]},\u0022capabilities\u0022:{\u0022usesExec\u0022:true,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[\u0022clockwork\u0022,\u0022sidekiq\u0022]},\u0022bundledWith\u0022:\u00222.5.3\u0022}", + "sha256": "sha256:fd8d3da9563e77103a04ec6020d8ec87dd7d9fab2160a31e289035787895f15e" } ] }, @@ -35,6 +36,7 @@ "type": "gem", "usedByEntrypoint": false, "metadata": { + "capability.exec": "true", "capability.net": "true", "capability.scheduler": "clockwork;sidekiq", "capability.scheduler.clockwork": "true", @@ -64,6 +66,7 @@ "type": "gem", "usedByEntrypoint": true, "metadata": { + "capability.exec": "true", "capability.net": "true", "capability.scheduler": "clockwork;sidekiq", "capability.scheduler.clockwork": "true", @@ -94,6 +97,7 @@ "type": "gem", "usedByEntrypoint": false, "metadata": { + "capability.exec": "true", "capability.net": "true", "capability.scheduler": "clockwork;sidekiq", "capability.scheduler.clockwork": "true", @@ -120,6 +124,7 @@ "type": "gem", "usedByEntrypoint": false, "metadata": { + "capability.exec": "true", "capability.net": "true", "capability.scheduler": "clockwork;sidekiq", "capability.scheduler.clockwork": "true", @@ -150,6 +155,7 @@ "usedByEntrypoint": false, "metadata": { "artifact": "vendor/custom-bundle/cache/sidekiq-7.2.1.gem", + "capability.exec": "true", "capability.net": "true", "capability.scheduler": "clockwork;sidekiq", "capability.scheduler.clockwork": "true", @@ -180,6 +186,7 @@ "usedByEntrypoint": false, "metadata": { "artifact": "vendor/cache/sinatra-3.1.0.gem", + "capability.exec": "true", "capability.net": "true", "capability.scheduler": "clockwork;sidekiq", "capability.scheduler.clockwork": "true", diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/container-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/container-app/expected.json index 4acfdc715..9c7c7e5b4 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/container-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/container-app/expected.json @@ -12,6 +12,8 @@ "ruby.observation.capability.schedulers": "0", "ruby.observation.capability.serialization": "false", "ruby.observation.dependency_edges": "3", + "ruby.observation.entrypoints": "2", + "ruby.observation.native_extensions": "2", "ruby.observation.packages": "7", "ruby.observation.ruby_version": "3.2.0", "ruby.observation.runtime_edges": "3", @@ -23,8 +25,8 @@ "kind": "derived", "source": "ruby.observation", "locator": "document", - "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022mini_portile2\u0022,\u0022version\u0022:\u00222.8.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022nio4r\u0022,\u0022version\u0022:\u00222.5.9\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022nokogiri\u0022,\u0022version\u0022:\u00221.15.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022pg\u0022,\u0022version\u0022:\u00221.5.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022puma\u0022,\u0022version\u0022:\u00226.4.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022racc\u0022,\u0022version\u0022:\u00221.7.1\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rack\u0022,\u0022version\u0022:\u00223.0.8\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]}],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022app.rb\u0022,\u0022type\u0022:\u0022script\u0022,\u0022requiredGems\u0022:[\u0022nokogiri\u0022,\u0022pg\u0022,\u0022rack\u0022]},{\u0022path\u0022:\u0022config.ru\u0022,\u0022type\u0022:\u0022rack\u0022,\u0022requiredGems\u0022:[\u0022rack\u0022]}],\u0022dependencyEdges\u0022:[{\u0022from\u0022:\u0022pkg:gem/nokogiri@1.15.0\u0022,\u0022to\u0022:\u0022mini_portile2\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.8.2\u0022},{\u0022from\u0022:\u0022pkg:gem/nokogiri@1.15.0\u0022,\u0022to\u0022:\u0022racc\u0022,\u0022constraint\u0022:\u0022~\\u003E 1.4\u0022},{\u0022from\u0022:\u0022pkg:gem/puma@6.4.0\u0022,\u0022to\u0022:\u0022nio4r\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.0\u0022}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022nokogiri\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app.rb\u0022],\u0022entrypoints\u0022:[\u0022app.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022pg\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app.rb\u0022],\u0022entrypoints\u0022:[\u0022app.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rack\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app.rb\u0022,\u0022config.ru\u0022],\u0022entrypoints\u0022:[\u0022app.rb\u0022,\u0022config.ru\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022configs\u0022:[{\u0022name\u0022:\u0022puma\u0022,\u0022type\u0022:\u0022web-server\u0022,\u0022filePath\u0022:\u0022config/puma.rb\u0022,\u0022settings\u0022:{\u0022preload_app\u0022:\u0022true\u0022}}],\u0022environment\u0022:{\u0022rubyVersion\u0022:\u00223.2.0\u0022,\u0022bundlerVersion\u0022:\u00222.4.22\u0022,\u0022lockfiles\u0022:[\u0022Gemfile.lock\u0022],\u0022rubyVersionSources\u0022:[{\u0022version\u0022:\u00223.2.0\u0022,\u0022source\u0022:\u0022.ruby-version\u0022,\u0022sourceType\u0022:\u0022ruby-version\u0022},{\u0022version\u0022:\u00223.2.0\u0022,\u0022source\u0022:\u0022.tool-versions\u0022,\u0022sourceType\u0022:\u0022tool-versions\u0022},{\u0022version\u0022:\u00223.2.0\u0022,\u0022source\u0022:\u0022Gemfile\u0022,\u0022sourceType\u0022:\u0022gemfile\u0022}],\u0022webServers\u0022:[{\u0022serverType\u0022:\u0022puma\u0022,\u0022configPath\u0022:\u0022config/puma.rb\u0022,\u0022settings\u0022:{\u0022preload_app\u0022:\u0022true\u0022}}]},\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:false,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]},\u0022bundledWith\u0022:\u00222.4.22\u0022}", - "sha256": "sha256:d5c7da885e1d05805981e2080c9023cd653ed464e993d5e48de6b9f55334eca7" + "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022mini_portile2\u0022,\u0022version\u0022:\u00222.8.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022nio4r\u0022,\u0022version\u0022:\u00222.5.9\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022nokogiri\u0022,\u0022version\u0022:\u00221.15.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022pg\u0022,\u0022version\u0022:\u00221.5.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022puma\u0022,\u0022version\u0022:\u00226.4.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022racc\u0022,\u0022version\u0022:\u00221.7.1\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rack\u0022,\u0022version\u0022:\u00223.0.8\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]}],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022app.rb\u0022,\u0022type\u0022:\u0022script\u0022,\u0022requiredGems\u0022:[\u0022nokogiri\u0022,\u0022pg\u0022,\u0022rack\u0022]},{\u0022path\u0022:\u0022config.ru\u0022,\u0022type\u0022:\u0022rack\u0022,\u0022requiredGems\u0022:[\u0022rack\u0022]}],\u0022dependencyEdges\u0022:[{\u0022from\u0022:\u0022pkg:gem/nokogiri@1.15.0\u0022,\u0022to\u0022:\u0022mini_portile2\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.8.2\u0022},{\u0022from\u0022:\u0022pkg:gem/nokogiri@1.15.0\u0022,\u0022to\u0022:\u0022racc\u0022,\u0022constraint\u0022:\u0022~\\u003E 1.4\u0022},{\u0022from\u0022:\u0022pkg:gem/puma@6.4.0\u0022,\u0022to\u0022:\u0022nio4r\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.0\u0022}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022nokogiri\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app.rb\u0022],\u0022entrypoints\u0022:[\u0022app.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022pg\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app.rb\u0022],\u0022entrypoints\u0022:[\u0022app.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rack\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app.rb\u0022,\u0022config.ru\u0022],\u0022entrypoints\u0022:[\u0022app.rb\u0022,\u0022config.ru\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022configs\u0022:[{\u0022name\u0022:\u0022puma\u0022,\u0022type\u0022:\u0022web-server\u0022,\u0022filePath\u0022:\u0022config/puma.rb\u0022,\u0022settings\u0022:{\u0022preload_app\u0022:\u0022true\u0022}}],\u0022environment\u0022:{\u0022rubyVersion\u0022:\u00223.2.0\u0022,\u0022bundlerVersion\u0022:\u00222.4.22\u0022,\u0022lockfiles\u0022:[\u0022Gemfile.lock\u0022],\u0022rubyVersionSources\u0022:[{\u0022version\u0022:\u00223.2.0\u0022,\u0022source\u0022:\u0022.ruby-version\u0022,\u0022sourceType\u0022:\u0022ruby-version\u0022},{\u0022version\u0022:\u00223.2.0\u0022,\u0022source\u0022:\u0022.tool-versions\u0022,\u0022sourceType\u0022:\u0022tool-versions\u0022},{\u0022version\u0022:\u00223.2.0\u0022,\u0022source\u0022:\u0022Gemfile\u0022,\u0022sourceType\u0022:\u0022gemfile\u0022}],\u0022webServers\u0022:[{\u0022serverType\u0022:\u0022puma\u0022,\u0022configPath\u0022:\u0022config/puma.rb\u0022,\u0022settings\u0022:{\u0022preload_app\u0022:\u0022true\u0022}}],\u0022nativeExtensions\u0022:[{\u0022gemName\u0022:\u0022nokogiri\u0022,\u0022gemVersion\u0022:\u00221.15.0\u0022,\u0022extensionPath\u0022:\u0022layers/ruby/usr/local/lib/ruby/gems/3.2.0/gems/nokogiri-1.15.0/lib/nokogiri/nokogiri.so\u0022,\u0022extensionType\u0022:\u0022so\u0022},{\u0022gemName\u0022:\u0022pg\u0022,\u0022gemVersion\u0022:\u00221.5.4\u0022,\u0022extensionPath\u0022:\u0022layers/ruby/usr/local/lib/ruby/gems/3.2.0/gems/pg-1.5.4/lib/pg_ext.so\u0022,\u0022extensionType\u0022:\u0022so\u0022}]},\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:false,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]},\u0022bundledWith\u0022:\u00222.4.22\u0022}", + "sha256": "sha256:59ac6bb07675933964aae184f8f581dfd33a334126e96f16fd0b1943374897c2" } ] }, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/legacy-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/legacy-app/expected.json index fe51488c7..dc417b27a 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/legacy-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/legacy-app/expected.json @@ -1 +1,28 @@ -[] +[ + { + "analyzerId": "ruby", + "componentKey": "observation::ruby", + "name": "Ruby Observation Summary", + "type": "ruby-observation", + "usedByEntrypoint": false, + "metadata": { + "ruby.observation.capability.exec": "true", + "ruby.observation.capability.net": "true", + "ruby.observation.capability.schedulers": "0", + "ruby.observation.capability.serialization": "false", + "ruby.observation.dependency_edges": "0", + "ruby.observation.entrypoints": "1", + "ruby.observation.packages": "0", + "ruby.observation.runtime_edges": "0" + }, + "evidence": [ + { + "kind": "derived", + "source": "ruby.observation", + "locator": "document", + "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022app.rb\u0022,\u0022type\u0022:\u0022script\u0022}],\u0022dependencyEdges\u0022:[],\u0022runtimeEdges\u0022:[],\u0022environment\u0022:{},\u0022capabilities\u0022:{\u0022usesExec\u0022:true,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]}}", + "sha256": "sha256:60173f2b841ca2b231eb293dd6ccc9d7ed39ea77a0e86ed887111f84861092ed" + } + ] + } +] \ No newline at end of file diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/rails-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/rails-app/expected.json index 23424f57a..d8ae4f13d 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/rails-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/rails-app/expected.json @@ -12,6 +12,7 @@ "ruby.observation.capability.schedulers": "0", "ruby.observation.capability.serialization": "false", "ruby.observation.dependency_edges": "13", + "ruby.observation.entrypoints": "3", "ruby.observation.packages": "18", "ruby.observation.ruby_version": "3.2.0", "ruby.observation.runtime_edges": "3" diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/simple-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/simple-app/expected.json index 88f5d2f1d..4239a6a6c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/simple-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/simple-app/expected.json @@ -13,6 +13,7 @@ "ruby.observation.capability.schedulers": "1", "ruby.observation.capability.serialization": "true", "ruby.observation.dependency_edges": "4", + "ruby.observation.entrypoints": "4", "ruby.observation.packages": "11", "ruby.observation.ruby_version": "3.1.2", "ruby.observation.runtime_edges": "3" diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/sinatra-app/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/sinatra-app/expected.json index 094c05d25..70187dbd0 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/sinatra-app/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/Fixtures/lang/ruby/sinatra-app/expected.json @@ -12,6 +12,7 @@ "ruby.observation.capability.schedulers": "0", "ruby.observation.capability.serialization": "false", "ruby.observation.dependency_edges": "12", + "ruby.observation.entrypoints": "2", "ruby.observation.packages": "11", "ruby.observation.runtime_edges": "2" }, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests.csproj index a691f5c81..dbd682a36 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests/StellaOps.Scanner.Analyzers.Lang.Ruby.Tests.csproj @@ -14,12 +14,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/deno/full/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/deno/full/expected.json index b6646efa7..93b7cf6f0 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/deno/full/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/deno/full/expected.json @@ -138,15 +138,17 @@ "metadata": { "deno.container.identifier": "vendor-", "deno.container.kind": "vendor", + "deno.container.layerDigest": "deadbeef", "deno.container.meta.alias": "vendor-", - "deno.container.meta.path": "/vendor" + "deno.container.meta.path": "/layers/sha256-deadbeef/fs/vendor" }, "evidence": [ { "kind": "metadata", "source": "deno.container", "locator": "Vendor", - "value": "vendor-" + "value": "vendor-", + "sha256": "deadbeef" } ] }, @@ -159,17 +161,15 @@ "metadata": { "deno.container.identifier": "vendor-", "deno.container.kind": "vendor", - "deno.container.layerDigest": "deadbeef", "deno.container.meta.alias": "vendor-", - "deno.container.meta.path": "/layers/sha256-deadbeef/fs/vendor" + "deno.container.meta.path": "/vendor" }, "evidence": [ { "kind": "metadata", "source": "deno.container", "locator": "Vendor", - "value": "vendor-", - "sha256": "deadbeef" + "value": "vendor-" } ] }, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/basic/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/basic/expected.json index 27b5e8265..93c9c2fe0 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/basic/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/basic/expected.json @@ -6,10 +6,14 @@ "type": "ruby-observation", "usedByEntrypoint": false, "metadata": { + "ruby.observation.bundler_version": "2.5.10", "ruby.observation.capability.exec": "true", "ruby.observation.capability.net": "true", + "ruby.observation.capability.scheduler_list": "activejob;clockwork;resque;sidekiq", "ruby.observation.capability.schedulers": "4", "ruby.observation.capability.serialization": "true", + "ruby.observation.dependency_edges": "1", + "ruby.observation.entrypoints": "1", "ruby.observation.packages": "3", "ruby.observation.runtime_edges": "3" }, @@ -18,8 +22,8 @@ "kind": "derived", "source": "ruby.observation", "locator": "document", - "value": "{\u0022packages\u0022:[{\u0022name\u0022:\u0022custom-gem\u0022,\u0022version\u0022:\u00221.0.0\u0022,\u0022source\u0022:\u0022vendor-cache\u0022,\u0022declaredOnly\u0022:false,\u0022artifact\u0022:\u0022vendor/cache/custom-gem-1.0.0.gem\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022puma\u0022,\u0022version\u0022:\u00226.4.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rake\u0022,\u0022version\u0022:\u002213.1.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022custom-gem\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022puma\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rake\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022capabilities\u0022:{\u0022usesExec\u0022:true,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:true,\u0022jobSchedulers\u0022:[\u0022activejob\u0022,\u0022clockwork\u0022,\u0022resque\u0022,\u0022sidekiq\u0022]}}", - "sha256": "sha256:3818fd050909977a44167565a419a307777bc38998ad49d6a41c054982c6f46e" + "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022custom-gem\u0022,\u0022version\u0022:\u00221.0.0\u0022,\u0022source\u0022:\u0022vendor-cache\u0022,\u0022declaredOnly\u0022:false,\u0022artifact\u0022:\u0022vendor/cache/custom-gem-1.0.0.gem\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022puma\u0022,\u0022version\u0022:\u00226.4.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rake\u0022,\u0022version\u0022:\u002213.1.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]}],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022app/main.rb\u0022,\u0022type\u0022:\u0022script\u0022,\u0022requiredGems\u0022:[\u0022custom-gem\u0022,\u0022puma\u0022,\u0022rake\u0022]}],\u0022dependencyEdges\u0022:[{\u0022from\u0022:\u0022pkg:gem/puma@6.4.2\u0022,\u0022to\u0022:\u0022nio4r\u0022,\u0022constraint\u0022:\u0022~\\u003E 2.0\u0022}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022custom-gem\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022puma\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rake\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022jobs\u0022:[{\u0022name\u0022:\u0022activejob\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022activejob\u0022},{\u0022name\u0022:\u0022clockwork\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022clockwork\u0022},{\u0022name\u0022:\u0022resque\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022resque\u0022},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022type\u0022:\u0022scheduler\u0022,\u0022scheduler\u0022:\u0022sidekiq\u0022}],\u0022environment\u0022:{\u0022bundlerVersion\u0022:\u00222.5.10\u0022,\u0022lockfiles\u0022:[\u0022Gemfile.lock\u0022],\u0022frameworks\u0022:[\u0022activejob\u0022,\u0022clockwork\u0022,\u0022resque\u0022,\u0022sidekiq\u0022]},\u0022capabilities\u0022:{\u0022usesExec\u0022:true,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:true,\u0022jobSchedulers\u0022:[\u0022activejob\u0022,\u0022clockwork\u0022,\u0022resque\u0022,\u0022sidekiq\u0022]},\u0022bundledWith\u0022:\u00222.5.10\u0022}", + "sha256": "sha256:260608f69ac45a4563892966a9146278a237ca3c79cc798511713213ed91f31d" } ] }, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/git-sources/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/git-sources/expected.json index ec51e0b85..961f191f3 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/git-sources/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/git-sources/expected.json @@ -6,10 +6,13 @@ "type": "ruby-observation", "usedByEntrypoint": false, "metadata": { + "ruby.observation.bundler_version": "2.5.10", "ruby.observation.capability.exec": "false", "ruby.observation.capability.net": "true", "ruby.observation.capability.schedulers": "0", "ruby.observation.capability.serialization": "false", + "ruby.observation.dependency_edges": "2", + "ruby.observation.entrypoints": "1", "ruby.observation.packages": "5", "ruby.observation.runtime_edges": "3" }, @@ -18,8 +21,8 @@ "kind": "derived", "source": "ruby.observation", "locator": "document", - "value": "{\u0022packages\u0022:[{\u0022name\u0022:\u0022git-gem\u0022,\u0022version\u0022:\u00220.5.0\u0022,\u0022source\u0022:\u0022git:https://github.com/example/git-gem.git@0123456789abcdef0123456789abcdef01234567\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022httparty\u0022,\u0022version\u0022:\u00220.21.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022multi_xml\u0022,\u0022version\u0022:\u00220.6.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022path-gem\u0022,\u0022version\u0022:\u00222.1.3\u0022,\u0022source\u0022:\u0022vendor-cache\u0022,\u0022declaredOnly\u0022:false,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022artifact\u0022:\u0022vendor/cache/path-gem-2.1.3.gem\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rake\u0022,\u0022version\u0022:\u002213.1.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022git-gem\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022httparty\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022path-gem\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]}}", - "sha256": "sha256:1cd5eb20a226916b9d1acbfc7182845a3ebca8284c7f558b23b7e87395e0a2c2" + "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022git-gem\u0022,\u0022version\u0022:\u00220.5.0\u0022,\u0022source\u0022:\u0022git:https://github.com/example/git-gem.git@0123456789abcdef0123456789abcdef01234567\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022httparty\u0022,\u0022version\u0022:\u00220.21.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022multi_xml\u0022,\u0022version\u0022:\u00220.6.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022path-gem\u0022,\u0022version\u0022:\u00222.1.3\u0022,\u0022source\u0022:\u0022vendor-cache\u0022,\u0022declaredOnly\u0022:false,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022artifact\u0022:\u0022vendor/cache/path-gem-2.1.3.gem\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rake\u0022,\u0022version\u0022:\u002213.1.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]}],\u0022entrypoints\u0022:[{\u0022path\u0022:\u0022app/main.rb\u0022,\u0022type\u0022:\u0022script\u0022,\u0022requiredGems\u0022:[\u0022git-gem\u0022,\u0022httparty\u0022,\u0022path-gem\u0022]}],\u0022dependencyEdges\u0022:[{\u0022from\u0022:\u0022pkg:gem/httparty@0.21.0\u0022,\u0022to\u0022:\u0022multi_xml\u0022,\u0022constraint\u0022:\u0022~\\u003E 0.5\u0022},{\u0022from\u0022:\u0022pkg:gem/path-gem@2.1.3\u0022,\u0022to\u0022:\u0022rake\u0022,\u0022constraint\u0022:\u0022~\\u003E 13.0\u0022}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022git-gem\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022httparty\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022path-gem\u0022,\u0022usedByEntrypoint\u0022:true,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[\u0022app/main.rb\u0022],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022environment\u0022:{\u0022bundlerVersion\u0022:\u00222.5.10\u0022,\u0022lockfiles\u0022:[\u0022Gemfile.lock\u0022]},\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:true,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]},\u0022bundledWith\u0022:\u00222.5.10\u0022}", + "sha256": "sha256:1c085acad0db516af25f986a033681de2b132adb719610fe29e88b0893447c25" } ] }, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/workspace/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/workspace/expected.json index de5ac5170..9bface154 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/workspace/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/ruby/workspace/expected.json @@ -6,10 +6,13 @@ "type": "ruby-observation", "usedByEntrypoint": false, "metadata": { + "ruby.observation.bundler_version": "2.5.10", "ruby.observation.capability.exec": "false", "ruby.observation.capability.net": "false", "ruby.observation.capability.schedulers": "0", "ruby.observation.capability.serialization": "false", + "ruby.observation.dependency_edges": "0", + "ruby.observation.entrypoints": "0", "ruby.observation.packages": "7", "ruby.observation.runtime_edges": "4" }, @@ -18,8 +21,8 @@ "kind": "derived", "source": "ruby.observation", "locator": "document", - "value": "{\u0022packages\u0022:[{\u0022name\u0022:\u0022api-gem\u0022,\u0022version\u0022:\u00220.1.0\u0022,\u0022source\u0022:\u0022apps\u0022,\u0022declaredOnly\u0022:false,\u0022artifact\u0022:\u0022apps/api/vendor/bundle/ruby/3.1.0/gems/api-gem-0.1.0\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022bootsnap\u0022,\u0022version\u0022:\u00221.18.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022apps/api/Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022pry\u0022,\u0022version\u0022:\u00221.0.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022development\u0022,\u0022test\u0022]},{\u0022name\u0022:\u0022puma\u0022,\u0022version\u0022:\u00226.4.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022console\u0022,\u0022production\u0022]},{\u0022name\u0022:\u0022rails\u0022,\u0022version\u0022:\u00227.1.3\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rubocop\u0022,\u0022version\u0022:\u00221.60.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022development\u0022,\u0022test\u0022]},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022version\u0022:\u00227.2.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022apps/api/Gemfile.lock\u0022,\u0022groups\u0022:[\u0022jobs\u0022]}],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022bootsnap\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022puma\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rails\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022sidekiq\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:false,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]}}", - "sha256": "sha256:6f9996b97be3dbbf3a18c2cb91624d45ddd16b2a374dd4a7f48049f5192114e2" + "value": "{\u0022$schema\u0022:\u0022stellaops.ruby.observation@1\u0022,\u0022packages\u0022:[{\u0022name\u0022:\u0022api-gem\u0022,\u0022version\u0022:\u00220.1.0\u0022,\u0022source\u0022:\u0022apps\u0022,\u0022declaredOnly\u0022:false,\u0022artifact\u0022:\u0022apps/api/vendor/bundle/ruby/3.1.0/gems/api-gem-0.1.0\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022bootsnap\u0022,\u0022version\u0022:\u00221.18.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022apps/api/Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022pry\u0022,\u0022version\u0022:\u00221.0.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022development\u0022,\u0022test\u0022]},{\u0022name\u0022:\u0022puma\u0022,\u0022version\u0022:\u00226.4.2\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022console\u0022,\u0022production\u0022]},{\u0022name\u0022:\u0022rails\u0022,\u0022version\u0022:\u00227.1.3\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022default\u0022]},{\u0022name\u0022:\u0022rubocop\u0022,\u0022version\u0022:\u00221.60.0\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022Gemfile.lock\u0022,\u0022groups\u0022:[\u0022development\u0022,\u0022test\u0022]},{\u0022name\u0022:\u0022sidekiq\u0022,\u0022version\u0022:\u00227.2.4\u0022,\u0022source\u0022:\u0022https://rubygems.org/\u0022,\u0022declaredOnly\u0022:true,\u0022lockfile\u0022:\u0022apps/api/Gemfile.lock\u0022,\u0022groups\u0022:[\u0022jobs\u0022]}],\u0022entrypoints\u0022:[],\u0022dependencyEdges\u0022:[],\u0022runtimeEdges\u0022:[{\u0022package\u0022:\u0022bootsnap\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022puma\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022rails\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]},{\u0022package\u0022:\u0022sidekiq\u0022,\u0022usedByEntrypoint\u0022:false,\u0022files\u0022:[\u0022app/main.rb\u0022],\u0022entrypoints\u0022:[],\u0022reasons\u0022:[\u0022require-static\u0022]}],\u0022environment\u0022:{\u0022bundlerVersion\u0022:\u00222.5.10\u0022,\u0022bundlePaths\u0022:[\u0022apps/api/vendor/bundle\u0022],\u0022gemfiles\u0022:[\u0022apps/api/Gemfile\u0022],\u0022lockfiles\u0022:[\u0022apps/api/Gemfile.lock\u0022,\u0022Gemfile.lock\u0022]},\u0022capabilities\u0022:{\u0022usesExec\u0022:false,\u0022usesNetwork\u0022:false,\u0022usesSerialization\u0022:false,\u0022jobSchedulers\u0022:[]},\u0022bundledWith\u0022:\u00222.5.10\u0022}", + "sha256": "sha256:b44788e3c6993f45cb372440f0e830677fe1b653ce4d6d468f1f5d2195e19fc5" } ] }, diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/fallback/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/fallback/expected.json index 548129573..6887c343a 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/fallback/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/fallback/expected.json @@ -1,13 +1,13 @@ [ { "analyzerId": "rust", - "componentKey": "bin::sha256:10f3c03766e4403be40add0467a2b2d07fd7006e4b8515ab88740ffa327ea775", + "componentKey": "bin::sha256:a037bf6e958bd6b2fdcc4a95c7dc6f7735730ae33d20819a056a5da050d05b8e", "name": "opaque_bin", "type": "bin", "usedByEntrypoint": true, "metadata": { "binary.path": "usr/local/bin/opaque_bin", - "binary.sha256": "10f3c03766e4403be40add0467a2b2d07fd7006e4b8515ab88740ffa327ea775", + "binary.sha256": "a037bf6e958bd6b2fdcc4a95c7dc6f7735730ae33d20819a056a5da050d05b8e", "provenance": "binary" }, "evidence": [ @@ -15,8 +15,8 @@ "kind": "file", "source": "binary", "locator": "usr/local/bin/opaque_bin", - "sha256": "10f3c03766e4403be40add0467a2b2d07fd7006e4b8515ab88740ffa327ea775" + "sha256": "a037bf6e958bd6b2fdcc4a95c7dc6f7735730ae33d20819a056a5da050d05b8e" } ] } -] \ No newline at end of file +] diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/heuristics/expected.json b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/heuristics/expected.json index 98fafef50..09881892b 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/heuristics/expected.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/rust/heuristics/expected.json @@ -7,7 +7,7 @@ "usedByEntrypoint": true, "metadata": { "binary.paths": "usr/local/bin/heuristic_app", - "binary.sha256": "4caf60c501a594b5d4b8d909b3e91fccc4447692b9e144f322a333255909310b", + "binary.sha256": "20cc78000c9ad10c9fe4be9d5458679d54298b170bbafc7198cf82700d06aa2c", "crate": "reqwest", "provenance": "heuristic" }, @@ -17,7 +17,7 @@ "source": "rust.heuristic", "locator": "usr/local/bin/heuristic_app", "value": "reqwest", - "sha256": "4caf60c501a594b5d4b8d909b3e91fccc4447692b9e144f322a333255909310b" + "sha256": "20cc78000c9ad10c9fe4be9d5458679d54298b170bbafc7198cf82700d06aa2c" } ] }, @@ -29,7 +29,7 @@ "usedByEntrypoint": true, "metadata": { "binary.paths": "usr/local/bin/heuristic_app", - "binary.sha256": "4caf60c501a594b5d4b8d909b3e91fccc4447692b9e144f322a333255909310b", + "binary.sha256": "20cc78000c9ad10c9fe4be9d5458679d54298b170bbafc7198cf82700d06aa2c", "crate": "serde", "provenance": "heuristic" }, @@ -39,7 +39,7 @@ "source": "rust.heuristic", "locator": "usr/local/bin/heuristic_app", "value": "serde", - "sha256": "4caf60c501a594b5d4b8d909b3e91fccc4447692b9e144f322a333255909310b" + "sha256": "20cc78000c9ad10c9fe4be9d5458679d54298b170bbafc7198cf82700d06aa2c" } ] }, @@ -51,7 +51,7 @@ "usedByEntrypoint": true, "metadata": { "binary.paths": "usr/local/bin/heuristic_app", - "binary.sha256": "4caf60c501a594b5d4b8d909b3e91fccc4447692b9e144f322a333255909310b", + "binary.sha256": "20cc78000c9ad10c9fe4be9d5458679d54298b170bbafc7198cf82700d06aa2c", "crate": "tokio", "provenance": "heuristic" }, @@ -61,7 +61,7 @@ "source": "rust.heuristic", "locator": "usr/local/bin/heuristic_app", "value": "tokio", - "sha256": "4caf60c501a594b5d4b8d909b3e91fccc4447692b9e144f322a333255909310b" + "sha256": "20cc78000c9ad10c9fe4be9d5458679d54298b170bbafc7198cf82700d06aa2c" } ] } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustFixtureBinaries.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustFixtureBinaries.cs new file mode 100644 index 000000000..fad9bd11c --- /dev/null +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustFixtureBinaries.cs @@ -0,0 +1,61 @@ +using System; +using System.IO; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Rust; + +internal static class RustFixtureBinaries +{ + private static readonly byte[] HeuristicBinary = + { + 0x7F, (byte)'E', (byte)'L', (byte)'F', + 0x02, 0x01, 0x01, 0x00, + (byte)'_', (byte)'Z', (byte)'N', (byte)'7', (byte)'r', (byte)'e', (byte)'q', (byte)'w', (byte)'e', (byte)'s', (byte)'t', + 0x00, + (byte)'_', (byte)'Z', (byte)'N', (byte)'5', (byte)'s', (byte)'e', (byte)'r', (byte)'d', (byte)'e', + 0x00, + (byte)'_', (byte)'Z', (byte)'N', (byte)'5', (byte)'t', (byte)'o', (byte)'k', (byte)'i', (byte)'o', + 0x00, + }; + + private static readonly byte[] OpaqueBinary = + { + 0x7F, (byte)'E', (byte)'L', (byte)'F', + 0x02, 0x01, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }; + + public static void EnsureHeuristicBinary(string fixturePath) + { + if (string.IsNullOrWhiteSpace(fixturePath)) + { + throw new ArgumentException("Fixture path is required.", nameof(fixturePath)); + } + + var path = Path.Combine(fixturePath, "usr", "local", "bin", "heuristic_app"); + WriteBinary(path, HeuristicBinary); + } + + public static void EnsureOpaqueBinary(string fixturePath) + { + if (string.IsNullOrWhiteSpace(fixturePath)) + { + throw new ArgumentException("Fixture path is required.", nameof(fixturePath)); + } + + var path = Path.Combine(fixturePath, "usr", "local", "bin", "opaque_bin"); + WriteBinary(path, OpaqueBinary); + } + + private static void WriteBinary(string path, byte[] content) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + File.WriteAllBytes(path, content); + } +} + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustHeuristicCoverageComparisonTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustHeuristicCoverageComparisonTests.cs index eb4398644..d28bcbd21 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustHeuristicCoverageComparisonTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustHeuristicCoverageComparisonTests.cs @@ -14,6 +14,7 @@ public sealed class RustHeuristicCoverageComparisonTests var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "rust", "heuristics"); var baselinePath = Path.Combine(fixturePath, "competitor-baseline.json"); + RustFixtureBinaries.EnsureHeuristicBinary(fixturePath); var analyzers = new ILanguageAnalyzer[] { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustLanguageAnalyzerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustLanguageAnalyzerTests.cs index 98543eec8..7774839a0 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustLanguageAnalyzerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Rust/RustLanguageAnalyzerTests.cs @@ -64,6 +64,7 @@ public sealed class RustLanguageAnalyzerTests var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "rust", "heuristics"); var goldenPath = Path.Combine(fixturePath, "expected.json"); + RustFixtureBinaries.EnsureHeuristicBinary(fixturePath); var usageHints = new LanguageUsageHints(new[] { Path.Combine(fixturePath, "usr/local/bin/heuristic_app") @@ -88,6 +89,7 @@ public sealed class RustLanguageAnalyzerTests var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "rust", "fallback"); var goldenPath = Path.Combine(fixturePath, "expected.json"); + RustFixtureBinaries.EnsureOpaqueBinary(fixturePath); var usageHints = new LanguageUsageHints(new[] { Path.Combine(fixturePath, "usr/local/bin/opaque_bin") diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj index 38b66bf75..6c172be10 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj @@ -18,12 +18,10 @@ - - diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs index 829942451..872c74023 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs @@ -94,6 +94,56 @@ public static class JavaFixtureBuilder libBuffer.CopyTo(libEntryStream); }); + public static string CreateSpringBootFatJarWithEmbeddedMavenLibrary(string rootDirectory, string relativePath = "apps/app-fat.jar") + => CreateJar(rootDirectory, relativePath, static archive => + { + var pomEntry = archive.CreateEntry("META-INF/maven/com.example/app-fat/pom.properties", CompressionLevel.NoCompression); + pomEntry.LastWriteTime = DefaultTimestamp; + using (var writer = new StreamWriter(pomEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("# Test pom.properties"); + writer.WriteLine("groupId=com.example"); + writer.WriteLine("artifactId=app-fat"); + writer.WriteLine("version=1.0.0"); + writer.WriteLine("name=App Fat"); + writer.WriteLine("packaging=jar"); + } + + var manifestEntry = archive.CreateEntry("META-INF/MANIFEST.MF", CompressionLevel.NoCompression); + manifestEntry.LastWriteTime = DefaultTimestamp; + using (var writer = new StreamWriter(manifestEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("Manifest-Version: 1.0"); + writer.WriteLine("Main-Class: org.springframework.boot.loader.JarLauncher"); + writer.WriteLine("Implementation-Title: App Fat"); + writer.WriteLine("Implementation-Version: 1.0.0"); + writer.WriteLine("Implementation-Vendor: Example Corp"); + writer.WriteLine(); + } + + using var libBuffer = new MemoryStream(); + using (var nested = new ZipArchive(libBuffer, ZipArchiveMode.Create, leaveOpen: true)) + { + var libPomEntry = nested.CreateEntry("META-INF/maven/com.example/embedded-lib/pom.properties", CompressionLevel.NoCompression); + libPomEntry.LastWriteTime = DefaultTimestamp; + using (var writer = new StreamWriter(libPomEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("# Test pom.properties"); + writer.WriteLine("groupId=com.example"); + writer.WriteLine("artifactId=embedded-lib"); + writer.WriteLine("version=2.1.0"); + writer.WriteLine("name=Embedded Lib"); + writer.WriteLine("packaging=jar"); + } + } + + libBuffer.Position = 0; + var libEntry = archive.CreateEntry("BOOT-INF/lib/embedded-lib.jar", CompressionLevel.NoCompression); + libEntry.LastWriteTime = DefaultTimestamp; + using var libEntryStream = libEntry.Open(); + libBuffer.CopyTo(libEntryStream); + }); + public static string CreateWarArchive(string rootDirectory, string relativePath = "apps/sample.war") => CreateJar(rootDirectory, relativePath, static archive => { @@ -135,6 +185,83 @@ public static class JavaFixtureBuilder libBuffer.CopyTo(libStream); }); + public static string CreateWarArchiveWithEmbeddedMavenLibrary(string rootDirectory, string relativePath = "apps/demo-war.war") + => CreateJar(rootDirectory, relativePath, static archive => + { + var pomEntry = archive.CreateEntry("META-INF/maven/com.example/demo-war/pom.properties", CompressionLevel.NoCompression); + pomEntry.LastWriteTime = DefaultTimestamp; + using (var writer = new StreamWriter(pomEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("# Test pom.properties"); + writer.WriteLine("groupId=com.example"); + writer.WriteLine("artifactId=demo-war"); + writer.WriteLine("version=1.0.0"); + writer.WriteLine("name=Demo War"); + writer.WriteLine("packaging=war"); + } + + var manifestEntry = archive.CreateEntry("META-INF/MANIFEST.MF", CompressionLevel.NoCompression); + manifestEntry.LastWriteTime = DefaultTimestamp; + using (var manifestWriter = new StreamWriter(manifestEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + manifestWriter.WriteLine("Manifest-Version: 1.0"); + manifestWriter.WriteLine("Implementation-Title: Demo War"); + manifestWriter.WriteLine("Implementation-Version: 1.0.0"); + manifestWriter.WriteLine("Implementation-Vendor: Example Corp"); + manifestWriter.WriteLine(); + } + + using var libBuffer = new MemoryStream(); + using (var nested = new ZipArchive(libBuffer, ZipArchiveMode.Create, leaveOpen: true)) + { + var libPomEntry = nested.CreateEntry("META-INF/maven/com.example/web-lib/pom.properties", CompressionLevel.NoCompression); + libPomEntry.LastWriteTime = DefaultTimestamp; + using (var writer = new StreamWriter(libPomEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("# Test pom.properties"); + writer.WriteLine("groupId=com.example"); + writer.WriteLine("artifactId=web-lib"); + writer.WriteLine("version=3.0.0"); + writer.WriteLine("name=Web Lib"); + writer.WriteLine("packaging=jar"); + } + } + + libBuffer.Position = 0; + var libEntry = archive.CreateEntry("WEB-INF/lib/web-lib.jar", CompressionLevel.NoCompression); + libEntry.LastWriteTime = DefaultTimestamp; + using var libStream = libEntry.Open(); + libBuffer.CopyTo(libStream); + }); + + public static string CreatePomXmlOnlyJar(string rootDirectory, string relativePath = "libs/pomxml-only.jar") + => CreateJar(rootDirectory, relativePath, static archive => + { + var manifestEntry = archive.CreateEntry("META-INF/MANIFEST.MF", CompressionLevel.NoCompression); + manifestEntry.LastWriteTime = DefaultTimestamp; + using (var manifestWriter = new StreamWriter(manifestEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + manifestWriter.WriteLine("Manifest-Version: 1.0"); + manifestWriter.WriteLine("Implementation-Title: PomXml Only"); + manifestWriter.WriteLine("Implementation-Version: 1.2.3"); + manifestWriter.WriteLine("Implementation-Vendor: Example Corp"); + manifestWriter.WriteLine(); + } + + var pomXmlEntry = archive.CreateEntry("META-INF/maven/com.example/pomxml-only/pom.xml", CompressionLevel.NoCompression); + pomXmlEntry.LastWriteTime = DefaultTimestamp; + using (var writer = new StreamWriter(pomXmlEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine(""); + writer.WriteLine(" 4.0.0"); + writer.WriteLine(" com.example"); + writer.WriteLine(" pomxml-only"); + writer.WriteLine(" 1.2.3"); + writer.WriteLine(" PomXml Only"); + writer.WriteLine(""); + } + }); + public static string CreateMultiReleaseJar(string rootDirectory, string relativePath = "libs/mr.jar") => CreateJar(rootDirectory, relativePath, static archive => { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs index c29c4fff8..1d4f612c0 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionPublisherTests.cs @@ -1,3 +1,4 @@ +using System; using System.Threading.Tasks; using StellaOps.Scanner.Core.Tests.Fakes; using StellaOps.Scanner.Reachability; @@ -27,4 +28,16 @@ public class ReachabilityUnionPublisherTests Assert.NotNull(entry); Assert.True(entry!.Value.SizeBytes > 0); } + + private sealed class TempDir : IDisposable + { + public string Path { get; } = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "reach-union-pub-" + Guid.NewGuid().ToString("N")); + + public TempDir() => System.IO.Directory.CreateDirectory(Path); + + public void Dispose() + { + try { System.IO.Directory.Delete(Path, recursive: true); } catch { /* best effort */ } + } + } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs index 32143967f..50259f04c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/ReachabilityUnionWriterTests.cs @@ -20,15 +20,15 @@ public class ReachabilityUnionWriterTests var graph = new ReachabilityUnionGraph( Nodes: new[] { - new ReachabilityUnionNode("sym:dotnet:B", "dotnet", "method", display: "B"), - new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method", display: "A", + new ReachabilityUnionNode("sym:dotnet:B", "dotnet", "method", Display: "B"), + new ReachabilityUnionNode("sym:dotnet:A", "dotnet", "method", Display: "A", Source: new ReachabilitySource("static", "il", "file.cs:10"), Attributes: new Dictionary { { "visibility", "public" } }), }, Edges: new[] { - new ReachabilityUnionEdge("sym:dotnet:B", "sym:dotnet:A", "call", confidence: "high"), - new ReachabilityUnionEdge("sym:dotnet:A", "sym:dotnet:B", "call", confidence: "high"), + new ReachabilityUnionEdge("sym:dotnet:B", "sym:dotnet:A", "call", Confidence: "high"), + new ReachabilityUnionEdge("sym:dotnet:A", "sym:dotnet:B", "call", Confidence: "high"), }, RuntimeFacts: new[] { diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Replay/RecordModeAssemblerTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Replay/RecordModeAssemblerTests.cs index caa5c79d2..53f78c25c 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Replay/RecordModeAssemblerTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/Replay/RecordModeAssemblerTests.cs @@ -1,5 +1,6 @@ using System; using FluentAssertions; +using StellaOps.Cryptography; using StellaOps.Replay.Core; using StellaOps.Scanner.Core.Replay; using Xunit; @@ -16,7 +17,9 @@ public sealed class RecordModeAssemblerTests Scan = new ReplayScanMetadata { Id = "scan-1", Time = DateTimeOffset.UnixEpoch } }; - var assembler = new RecordModeAssembler(new FixedTimeProvider(new DateTimeOffset(2025, 11, 25, 12, 0, 0, TimeSpan.Zero))); + var assembler = new RecordModeAssembler( + CryptoHashFactory.CreateDefault(), + new FixedTimeProvider(new DateTimeOffset(2025, 11, 25, 12, 0, 0, TimeSpan.Zero))); var run = assembler.BuildRun("scan-1", manifest, "sha256:sbom", "findings-digest", vexDigest: "sha256:vex"); @@ -32,7 +35,9 @@ public sealed class RecordModeAssemblerTests [Fact] public void BuildBundles_ProducesDeterministicRecords() { - var assembler = new RecordModeAssembler(new FixedTimeProvider(DateTimeOffset.UnixEpoch)); + var assembler = new RecordModeAssembler( + CryptoHashFactory.CreateDefault(), + new FixedTimeProvider(DateTimeOffset.UnixEpoch)); var input = new ReplayBundleWriteResult("tar1", "z1", 10, 20, "cas://replay/zz/z1.tar.zst"); var output = new ReplayBundleWriteResult("tar2", "z2", 30, 40, "cas://replay/aa/z2.tar.zst"); diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj index 14699188d..a64b3b16f 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj +++ b/src/Scanner/__Tests/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj @@ -12,6 +12,9 @@ + + + diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/BinaryReachabilityLifterTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/BinaryReachabilityLifterTests.cs index c3bf5f375..a420906cc 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/BinaryReachabilityLifterTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/BinaryReachabilityLifterTests.cs @@ -530,9 +530,9 @@ public class BinaryReachabilityLifterTests var coff = peHeaderOffset + 4; WriteU16LE(buffer, coff + 0, 0x8664); // Machine WriteU16LE(buffer, coff + 2, 1); // NumberOfSections - WriteU32LE(buffer, coff + 16, 0); // NumberOfSymbols - WriteU16LE(buffer, coff + 16 + 4, (ushort)optionalHeaderSize); // SizeOfOptionalHeader - WriteU16LE(buffer, coff + 16 + 6, 0x22); // Characteristics + WriteU32LE(buffer, coff + 12, 0); // NumberOfSymbols + WriteU16LE(buffer, coff + 16, (ushort)optionalHeaderSize); // SizeOfOptionalHeader + WriteU16LE(buffer, coff + 18, 0x22); // Characteristics var opt = peHeaderOffset + 24; WriteU16LE(buffer, opt + 0, 0x20b); // PE32+ diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs index 63772e5dd..e86ef94cd 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/FakeFileContentAddressableStore.cs @@ -11,6 +11,9 @@ internal sealed class FakeFileContentAddressableStore : IFileContentAddressableS { private readonly ConcurrentDictionary store = new(); + public byte[]? GetBytes(string key) + => store.TryGetValue(key, out var bytes) ? bytes : null; + public ValueTask TryGetAsync(string sha256, CancellationToken cancellationToken = default) { if (store.TryGetValue(sha256, out var bytes)) diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/RichGraphPublisherTests.cs b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/RichGraphPublisherTests.cs index c50ad623b..6cb90e340 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/RichGraphPublisherTests.cs +++ b/src/Scanner/__Tests/StellaOps.Scanner.Reachability.Tests/RichGraphPublisherTests.cs @@ -1,3 +1,5 @@ +using System; +using System.Text.Json; using System.Threading.Tasks; using StellaOps.Cryptography; using StellaOps.Scanner.Reachability; @@ -24,6 +26,46 @@ public class RichGraphPublisherTests Assert.Contains(":", result.GraphHash); // hash format: algorithm:digest Assert.StartsWith("cas://reachability/graphs/", result.CasUri); + Assert.StartsWith("cas://reachability/graphs/", result.DsseCasUri); + Assert.EndsWith(".dsse", result.DsseCasUri, StringComparison.Ordinal); + Assert.StartsWith("sha256:", result.DsseDigest, StringComparison.Ordinal); Assert.Equal(1, result.NodeCount); + + var casKey = result.CasUri[(result.CasUri.LastIndexOf('/') + 1)..]; + var dsseKey = $"{casKey}.dsse"; + var dsseBytes = cas.GetBytes(dsseKey); + Assert.NotNull(dsseBytes); + + using var dsseDoc = JsonDocument.Parse(dsseBytes!); + Assert.Equal( + "application/vnd.stellaops.graph.predicate+json", + dsseDoc.RootElement.GetProperty("payloadType").GetString()); + + var payloadBase64Url = dsseDoc.RootElement.GetProperty("payload").GetString(); + Assert.False(string.IsNullOrWhiteSpace(payloadBase64Url)); + + var payloadBytes = Base64UrlDecode(payloadBase64Url!); + using var payloadDoc = JsonDocument.Parse(payloadBytes); + Assert.Equal( + result.GraphHash, + payloadDoc.RootElement.GetProperty("hashes").GetProperty("graphHash").GetString()); + Assert.Equal( + result.CasUri, + payloadDoc.RootElement.GetProperty("cas").GetProperty("location").GetString()); + + var signature = dsseDoc.RootElement.GetProperty("signatures")[0]; + Assert.Equal("scanner-deterministic", signature.GetProperty("keyid").GetString()); + } + + private static byte[] Base64UrlDecode(string value) + { + var normalized = value.Replace('-', '+').Replace('_', '/'); + var remainder = normalized.Length % 4; + if (remainder != 0) + { + normalized = normalized.PadRight(normalized.Length + (4 - remainder), '='); + } + + return Convert.FromBase64String(normalized); } } diff --git a/src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json b/src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json index c39f33b67..ca44d89e1 100644 --- a/src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json +++ b/src/Scanner/__Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json @@ -19,8 +19,8 @@ "org.stellaops.sbom.kind": "inventory", "org.stellaops.sbom.format": "cyclonedx-json", "org.stellaops.provenance.status": "pending", - "org.stellaops.provenance.dsse.sha256": "sha256:35ab4784f3bad40bb0063b522939ac729cf43d2012059947c0e56475d682c05e", - "org.stellaops.provenance.nonce": "5e13230e3dcbc8be996d8132d92e8826", + "org.stellaops.provenance.dsse.sha256": "sha256:1b364a6b888d580feb8565f7b6195b24535ca8201b4bcac58da063b32c47220d", + "org.stellaops.provenance.nonce": "a608acf859cd58a8389816b8d9eb2a07", "org.stellaops.license.id": "lic-123", "org.opencontainers.image.title": "sample.cdx.json", "org.stellaops.repository": "git.stella-ops.org/stellaops" @@ -28,8 +28,8 @@ }, "provenance": { "status": "pending", - "expectedDsseSha256": "sha256:35ab4784f3bad40bb0063b522939ac729cf43d2012059947c0e56475d682c05e", - "nonce": "5e13230e3dcbc8be996d8132d92e8826", + "expectedDsseSha256": "sha256:1b364a6b888d580feb8565f7b6195b24535ca8201b4bcac58da063b32c47220d", + "nonce": "a608acf859cd58a8389816b8d9eb2a07", "attestorUri": "https://attestor.local/api/v1/provenance", "predicateType": "https://slsa.dev/provenance/v1" }, diff --git a/src/Signals/StellaOps.Signals/Models/ReachabilityStore/CallEdgeDocument.cs b/src/Signals/StellaOps.Signals/Models/ReachabilityStore/CallEdgeDocument.cs new file mode 100644 index 000000000..597ed215a --- /dev/null +++ b/src/Signals/StellaOps.Signals/Models/ReachabilityStore/CallEdgeDocument.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Signals.Models.ReachabilityStore; + +public sealed class CallEdgeDocument +{ + public string Id { get; set; } = string.Empty; + + public string GraphHash { get; set; } = string.Empty; + + public string SourceId { get; set; } = string.Empty; + + public string TargetId { get; set; } = string.Empty; + + public string Type { get; set; } = string.Empty; + + public string? Purl { get; set; } + + public string? SymbolDigest { get; set; } + + public List? Candidates { get; set; } + + public double? Confidence { get; set; } + + public List? Evidence { get; set; } + + public DateTimeOffset IngestedAt { get; set; } +} + diff --git a/src/Signals/StellaOps.Signals/Models/ReachabilityStore/CveFuncHitDocument.cs b/src/Signals/StellaOps.Signals/Models/ReachabilityStore/CveFuncHitDocument.cs new file mode 100644 index 000000000..fb5c358a4 --- /dev/null +++ b/src/Signals/StellaOps.Signals/Models/ReachabilityStore/CveFuncHitDocument.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Signals.Models.ReachabilityStore; + +public sealed class CveFuncHitDocument +{ + public string Id { get; set; } = string.Empty; + + public string SubjectKey { get; set; } = string.Empty; + + public string CveId { get; set; } = string.Empty; + + public string GraphHash { get; set; } = string.Empty; + + public string? Purl { get; set; } + + public string? SymbolDigest { get; set; } + + public bool Reachable { get; set; } + + public double? Confidence { get; set; } + + public string? LatticeState { get; set; } + + public List? EvidenceUris { get; set; } + + public DateTimeOffset ComputedAt { get; set; } +} + diff --git a/src/Signals/StellaOps.Signals/Models/ReachabilityStore/FuncNodeDocument.cs b/src/Signals/StellaOps.Signals/Models/ReachabilityStore/FuncNodeDocument.cs new file mode 100644 index 000000000..73fa354bd --- /dev/null +++ b/src/Signals/StellaOps.Signals/Models/ReachabilityStore/FuncNodeDocument.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Signals.Models.ReachabilityStore; + +public sealed class FuncNodeDocument +{ + public string Id { get; set; } = string.Empty; + + public string GraphHash { get; set; } = string.Empty; + + public string SymbolId { get; set; } = string.Empty; + + public string Name { get; set; } = string.Empty; + + public string Kind { get; set; } = string.Empty; + + public string? Namespace { get; set; } + + public string? File { get; set; } + + public int? Line { get; set; } + + public string? Purl { get; set; } + + public string? SymbolDigest { get; set; } + + public string? BuildId { get; set; } + + public string? CodeId { get; set; } + + public string? Language { get; set; } + + public List? Evidence { get; set; } + + public Dictionary? Analyzer { get; set; } + + public DateTimeOffset IngestedAt { get; set; } +} + diff --git a/src/Signals/StellaOps.Signals/Persistence/IReachabilityStoreRepository.cs b/src/Signals/StellaOps.Signals/Persistence/IReachabilityStoreRepository.cs new file mode 100644 index 000000000..fef3525d6 --- /dev/null +++ b/src/Signals/StellaOps.Signals/Persistence/IReachabilityStoreRepository.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Signals.Models; +using StellaOps.Signals.Models.ReachabilityStore; + +namespace StellaOps.Signals.Persistence; + +public interface IReachabilityStoreRepository +{ + Task UpsertGraphAsync( + string graphHash, + IReadOnlyCollection nodes, + IReadOnlyCollection edges, + CancellationToken cancellationToken); + + Task> GetFuncNodesByGraphAsync(string graphHash, CancellationToken cancellationToken); + + Task> GetCallEdgesByGraphAsync(string graphHash, CancellationToken cancellationToken); + + Task UpsertCveFuncHitsAsync(IReadOnlyCollection hits, CancellationToken cancellationToken); + + Task> GetCveFuncHitsBySubjectAsync( + string subjectKey, + string cveId, + CancellationToken cancellationToken); +} + diff --git a/src/Signals/StellaOps.Signals/Persistence/InMemoryReachabilityStoreRepository.cs b/src/Signals/StellaOps.Signals/Persistence/InMemoryReachabilityStoreRepository.cs new file mode 100644 index 000000000..d682427dd --- /dev/null +++ b/src/Signals/StellaOps.Signals/Persistence/InMemoryReachabilityStoreRepository.cs @@ -0,0 +1,250 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Signals.Models; +using StellaOps.Signals.Models.ReachabilityStore; + +namespace StellaOps.Signals.Persistence; + +internal sealed class InMemoryReachabilityStoreRepository : IReachabilityStoreRepository +{ + private readonly TimeProvider timeProvider; + private readonly ConcurrentDictionary funcNodes = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary callEdges = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary cveFuncHits = new(StringComparer.Ordinal); + + public InMemoryReachabilityStoreRepository(TimeProvider timeProvider) + { + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public Task UpsertGraphAsync( + string graphHash, + IReadOnlyCollection nodes, + IReadOnlyCollection edges, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(graphHash); + ArgumentNullException.ThrowIfNull(nodes); + ArgumentNullException.ThrowIfNull(edges); + + var normalizedGraphHash = graphHash.Trim(); + var now = timeProvider.GetUtcNow(); + + foreach (var node in nodes) + { + var document = ToFuncNodeDocument(normalizedGraphHash, node, now); + funcNodes[document.Id] = document; + } + + foreach (var edge in edges) + { + var document = ToCallEdgeDocument(normalizedGraphHash, edge, now); + callEdges[document.Id] = document; + } + + return Task.CompletedTask; + } + + public Task> GetFuncNodesByGraphAsync(string graphHash, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(graphHash)) + { + return Task.FromResult((IReadOnlyList)Array.Empty()); + } + + var normalizedGraphHash = graphHash.Trim(); + + var results = funcNodes.Values + .Where(doc => string.Equals(doc.GraphHash, normalizedGraphHash, StringComparison.Ordinal)) + .Select(Clone) + .OrderBy(doc => doc.SymbolId, StringComparer.Ordinal) + .ThenBy(doc => doc.Purl, StringComparer.Ordinal) + .ThenBy(doc => doc.SymbolDigest, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult((IReadOnlyList)results); + } + + public Task> GetCallEdgesByGraphAsync(string graphHash, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(graphHash)) + { + return Task.FromResult((IReadOnlyList)Array.Empty()); + } + + var normalizedGraphHash = graphHash.Trim(); + + var results = callEdges.Values + .Where(doc => string.Equals(doc.GraphHash, normalizedGraphHash, StringComparison.Ordinal)) + .Select(Clone) + .OrderBy(doc => doc.SourceId, StringComparer.Ordinal) + .ThenBy(doc => doc.TargetId, StringComparer.Ordinal) + .ThenBy(doc => doc.Type, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult((IReadOnlyList)results); + } + + public Task UpsertCveFuncHitsAsync(IReadOnlyCollection hits, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(hits); + + foreach (var hit in hits.Where(h => h is not null)) + { + if (string.IsNullOrWhiteSpace(hit.SubjectKey) || string.IsNullOrWhiteSpace(hit.CveId)) + { + continue; + } + + var id = BuildCveFuncHitId(hit.SubjectKey, hit.CveId, hit.Purl, hit.SymbolDigest); + var clone = Clone(hit); + clone.Id = id; + cveFuncHits[id] = clone; + } + + return Task.CompletedTask; + } + + public Task> GetCveFuncHitsBySubjectAsync( + string subjectKey, + string cveId, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(subjectKey) || string.IsNullOrWhiteSpace(cveId)) + { + return Task.FromResult((IReadOnlyList)Array.Empty()); + } + + var normalizedSubjectKey = subjectKey.Trim(); + var normalizedCve = cveId.Trim(); + + var results = cveFuncHits.Values + .Where(doc => + string.Equals(doc.SubjectKey, normalizedSubjectKey, StringComparison.Ordinal) && + string.Equals(doc.CveId, normalizedCve, StringComparison.OrdinalIgnoreCase)) + .Select(Clone) + .OrderBy(doc => doc.CveId, StringComparer.OrdinalIgnoreCase) + .ThenBy(doc => doc.Purl, StringComparer.Ordinal) + .ThenBy(doc => doc.SymbolDigest, StringComparer.Ordinal) + .ToList(); + + return Task.FromResult((IReadOnlyList)results); + } + + private static FuncNodeDocument ToFuncNodeDocument(string graphHash, CallgraphNode node, DateTimeOffset ingestedAt) + { + var symbolId = node.Id?.Trim() ?? string.Empty; + var id = BuildFuncNodeId(graphHash, symbolId); + + return new FuncNodeDocument + { + Id = id, + GraphHash = graphHash, + SymbolId = symbolId, + Name = node.Name?.Trim() ?? string.Empty, + Kind = node.Kind?.Trim() ?? string.Empty, + Namespace = node.Namespace?.Trim(), + File = node.File?.Trim(), + Line = node.Line, + Purl = node.Purl?.Trim(), + SymbolDigest = node.SymbolDigest?.Trim()?.ToLowerInvariant(), + BuildId = node.BuildId?.Trim(), + CodeId = node.CodeId?.Trim(), + Language = node.Language?.Trim(), + Evidence = node.Evidence?.Where(v => !string.IsNullOrWhiteSpace(v)).Select(v => v.Trim()).OrderBy(v => v, StringComparer.Ordinal).ToList(), + Analyzer = node.Analyzer is null + ? null + : node.Analyzer + .Where(kv => !string.IsNullOrWhiteSpace(kv.Key)) + .OrderBy(kv => kv.Key, StringComparer.Ordinal) + .ToDictionary(kv => kv.Key.Trim(), kv => kv.Value?.Trim(), StringComparer.Ordinal), + IngestedAt = ingestedAt + }; + } + + private static CallEdgeDocument ToCallEdgeDocument(string graphHash, CallgraphEdge edge, DateTimeOffset ingestedAt) + { + var sourceId = edge.SourceId?.Trim() ?? string.Empty; + var targetId = edge.TargetId?.Trim() ?? string.Empty; + var type = edge.Type?.Trim() ?? string.Empty; + var id = BuildCallEdgeId(graphHash, sourceId, targetId, type); + + return new CallEdgeDocument + { + Id = id, + GraphHash = graphHash, + SourceId = sourceId, + TargetId = targetId, + Type = type, + Purl = edge.Purl?.Trim(), + SymbolDigest = edge.SymbolDigest?.Trim()?.ToLowerInvariant(), + Candidates = edge.Candidates?.Where(v => !string.IsNullOrWhiteSpace(v)).Select(v => v.Trim()).OrderBy(v => v, StringComparer.Ordinal).ToList(), + Confidence = edge.Confidence, + Evidence = edge.Evidence?.Where(v => !string.IsNullOrWhiteSpace(v)).Select(v => v.Trim()).OrderBy(v => v, StringComparer.Ordinal).ToList(), + IngestedAt = ingestedAt + }; + } + + private static string BuildFuncNodeId(string graphHash, string symbolId) + => $"{graphHash}|{symbolId}"; + + private static string BuildCallEdgeId(string graphHash, string sourceId, string targetId, string type) + => $"{graphHash}|{sourceId}->{targetId}|{type}"; + + private static string BuildCveFuncHitId(string subjectKey, string cveId, string? purl, string? symbolDigest) + => $"{subjectKey.Trim()}|{cveId.Trim().ToUpperInvariant()}|{purl?.Trim() ?? string.Empty}|{symbolDigest?.Trim()?.ToLowerInvariant() ?? string.Empty}"; + + private static FuncNodeDocument Clone(FuncNodeDocument source) => new() + { + Id = source.Id, + GraphHash = source.GraphHash, + SymbolId = source.SymbolId, + Name = source.Name, + Kind = source.Kind, + Namespace = source.Namespace, + File = source.File, + Line = source.Line, + Purl = source.Purl, + SymbolDigest = source.SymbolDigest, + BuildId = source.BuildId, + CodeId = source.CodeId, + Language = source.Language, + Evidence = source.Evidence?.ToList(), + Analyzer = source.Analyzer is null ? null : new Dictionary(source.Analyzer, StringComparer.Ordinal), + IngestedAt = source.IngestedAt + }; + + private static CallEdgeDocument Clone(CallEdgeDocument source) => new() + { + Id = source.Id, + GraphHash = source.GraphHash, + SourceId = source.SourceId, + TargetId = source.TargetId, + Type = source.Type, + Purl = source.Purl, + SymbolDigest = source.SymbolDigest, + Candidates = source.Candidates?.ToList(), + Confidence = source.Confidence, + Evidence = source.Evidence?.ToList(), + IngestedAt = source.IngestedAt + }; + + private static CveFuncHitDocument Clone(CveFuncHitDocument source) => new() + { + Id = source.Id, + SubjectKey = source.SubjectKey, + CveId = source.CveId, + GraphHash = source.GraphHash, + Purl = source.Purl, + SymbolDigest = source.SymbolDigest, + Reachable = source.Reachable, + Confidence = source.Confidence, + LatticeState = source.LatticeState, + EvidenceUris = source.EvidenceUris?.ToList(), + ComputedAt = source.ComputedAt + }; +} diff --git a/src/Signals/StellaOps.Signals/Program.cs b/src/Signals/StellaOps.Signals/Program.cs index 34adfbbea..30026176d 100644 --- a/src/Signals/StellaOps.Signals/Program.cs +++ b/src/Signals/StellaOps.Signals/Program.cs @@ -132,6 +132,7 @@ builder.Services.AddSingleton(sp => return new ReachabilityFactCacheDecorator(inner, cache); }); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddHttpClient((sp, client) => { var opts = sp.GetRequiredService().Events.Router; diff --git a/src/Signals/StellaOps.Signals/Services/CallgraphIngestionService.cs b/src/Signals/StellaOps.Signals/Services/CallgraphIngestionService.cs index 5251fbb30..941595c1d 100644 --- a/src/Signals/StellaOps.Signals/Services/CallgraphIngestionService.cs +++ b/src/Signals/StellaOps.Signals/Services/CallgraphIngestionService.cs @@ -29,6 +29,7 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService private readonly ICallgraphParserResolver parserResolver; private readonly ICallgraphArtifactStore artifactStore; private readonly ICallgraphRepository repository; + private readonly IReachabilityStoreRepository reachabilityStore; private readonly ICallgraphNormalizationService normalizer; private readonly ILogger logger; private readonly SignalsOptions options; @@ -39,6 +40,7 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService ICallgraphParserResolver parserResolver, ICallgraphArtifactStore artifactStore, ICallgraphRepository repository, + IReachabilityStoreRepository reachabilityStore, ICallgraphNormalizationService normalizer, IOptions options, TimeProvider timeProvider, @@ -47,6 +49,7 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService this.parserResolver = parserResolver ?? throw new ArgumentNullException(nameof(parserResolver)); this.artifactStore = artifactStore ?? throw new ArgumentNullException(nameof(artifactStore)); this.repository = repository ?? throw new ArgumentNullException(nameof(repository)); + this.reachabilityStore = reachabilityStore ?? throw new ArgumentNullException(nameof(reachabilityStore)); this.normalizer = normalizer ?? throw new ArgumentNullException(nameof(normalizer)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); @@ -143,6 +146,12 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false); + await reachabilityStore.UpsertGraphAsync( + document.GraphHash, + document.Nodes, + document.Edges, + cancellationToken).ConfigureAwait(false); + logger.LogInformation( "Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.", document.Language, diff --git a/src/Signals/StellaOps.Signals/Services/ReachabilityScoringService.cs b/src/Signals/StellaOps.Signals/Services/ReachabilityScoringService.cs index e64399ad1..d6a034a10 100644 --- a/src/Signals/StellaOps.Signals/Services/ReachabilityScoringService.cs +++ b/src/Signals/StellaOps.Signals/Services/ReachabilityScoringService.cs @@ -157,8 +157,12 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService var finalScore = baseScore * (1 - pressurePenalty); var uncertaintyStates = MergeUncertaintyStates(existingFact?.Uncertainty?.States, unknownsCount, pressure, states.Count, computedAt); - var (uncertainty, aggregateTier) = BuildUncertaintyDocument(uncertaintyStates, baseScore, computedAt); - var riskScore = ComputeRiskScoreWithTiers(baseScore, uncertaintyStates, aggregateTier); + var (uncertainty, _, riskScore) = BuildUncertaintyDocument( + uncertaintyStates, + baseScore, + computedAt, + scoringOptions.UncertaintyEntropyMultiplier, + scoringOptions.UncertaintyBoostCeiling); var document = new ReachabilityFactDocument { @@ -252,14 +256,16 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService .ToList(); } - private static (UncertaintyDocument? Document, UncertaintyTier AggregateTier) BuildUncertaintyDocument( + private static (UncertaintyDocument? Document, UncertaintyTier AggregateTier, double RiskScore) BuildUncertaintyDocument( List states, double baseScore, - DateTimeOffset computedAt) + DateTimeOffset computedAt, + double entropyMultiplier, + double boostCeiling) { if (states.Count == 0) { - return (null, UncertaintyTier.T4); + return (null, UncertaintyTier.T4, baseScore); } // Calculate aggregate tier @@ -270,7 +276,12 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService var meanEntropy = states.Average(s => s.Entropy); // Calculate risk score with tier modifiers - var riskScore = UncertaintyTierCalculator.CalculateRiskScore(baseScore, aggregateTier, meanEntropy); + var riskScore = UncertaintyTierCalculator.CalculateRiskScore( + baseScore, + aggregateTier, + meanEntropy, + entropyMultiplier, + boostCeiling); var document = new UncertaintyDocument { @@ -280,7 +291,7 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService ComputedAt = computedAt }; - return (document, aggregateTier); + return (document, aggregateTier, riskScore); } private static UncertaintyStateDocument NormalizeState(UncertaintyStateDocument state) @@ -315,23 +326,6 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService }; } - private double ComputeRiskScoreWithTiers( - double baseScore, - IReadOnlyList uncertaintyStates, - UncertaintyTier aggregateTier) - { - var meanEntropy = uncertaintyStates.Count > 0 - ? uncertaintyStates.Average(s => s.Entropy) - : 0.0; - - return UncertaintyTierCalculator.CalculateRiskScore( - baseScore, - aggregateTier, - meanEntropy, - scoringOptions.UncertaintyEntropyMultiplier, - scoringOptions.UncertaintyBoostCeiling); - } - private static void ValidateRequest(ReachabilityRecomputeRequest request) { if (string.IsNullOrWhiteSpace(request.CallgraphId)) diff --git a/src/Signals/StellaOps.Signals/TASKS.md b/src/Signals/StellaOps.Signals/TASKS.md index 988137f5c..fa35dfa0e 100644 --- a/src/Signals/StellaOps.Signals/TASKS.md +++ b/src/Signals/StellaOps.Signals/TASKS.md @@ -4,4 +4,6 @@ This file mirrors sprint work for the Signals module. | Task ID | Sprint | Status | Notes | | --- | --- | --- | --- | -| `UNCERTAINTY-SCHEMA-401-024` | `docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md` | DOING | Add uncertainty states + entropy-derived `riskScore` to reachability facts and update events/tests. | +| `SIG-STORE-401-016` | `docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md` | DONE (2025-12-13) | Added reachability store repository APIs and models; callgraph ingestion now populates the store; Mongo index script at `ops/mongo/indices/reachability_store_indices.js`. | +| `UNCERTAINTY-SCHEMA-401-024` | `docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md` | DONE (2025-12-13) | Implemented uncertainty tiers and scoring integration; see `src/Signals/StellaOps.Signals/Lattice/UncertaintyTier.cs` and `src/Signals/StellaOps.Signals/Lattice/ReachabilityLattice.cs`. | +| `UNCERTAINTY-SCORER-401-025` | `docs/implplan/SPRINT_0401_0001_0001_reachability_evidence_chain.md` | DONE (2025-12-13) | Reachability risk score now uses configurable entropy weights and is aligned with `UncertaintyDocument.RiskScore`; tests cover tier/entropy scoring. | diff --git a/src/Signals/__Tests/StellaOps.Signals.Tests/CallgraphIngestionServiceTests.cs b/src/Signals/__Tests/StellaOps.Signals.Tests/CallgraphIngestionServiceTests.cs index e2ca71caa..55ad4c79f 100644 --- a/src/Signals/__Tests/StellaOps.Signals.Tests/CallgraphIngestionServiceTests.cs +++ b/src/Signals/__Tests/StellaOps.Signals.Tests/CallgraphIngestionServiceTests.cs @@ -32,10 +32,12 @@ public class CallgraphIngestionServiceTests var parser = new StubParser("java"); var resolver = new StubParserResolver(parser); var options = Microsoft.Extensions.Options.Options.Create(new SignalsOptions()); + var reachabilityStore = new InMemoryReachabilityStoreRepository(_timeProvider); var service = new CallgraphIngestionService( resolver, _artifactStore, _repository, + reachabilityStore, _normalizer, options, _timeProvider, @@ -70,6 +72,15 @@ public class CallgraphIngestionServiceTests stored.Metadata!["schemaVersion"].Should().Be("1.0"); stored.Metadata!["analyzer.name"].Should().Be("stub"); stored.Artifact.GraphHash.Should().Be(response.GraphHash); + + var storedNodes = await reachabilityStore.GetFuncNodesByGraphAsync(response.GraphHash, CancellationToken.None); + storedNodes.Should().HaveCount(1); + storedNodes[0].SymbolId.Should().Be("com/example/Foo.bar:(I)V"); + + var storedEdges = await reachabilityStore.GetCallEdgesByGraphAsync(response.GraphHash, CancellationToken.None); + storedEdges.Should().HaveCount(1); + storedEdges[0].SourceId.Should().Be("com/example/Foo.bar:(I)V"); + storedEdges[0].TargetId.Should().Be("com/example/Foo.bar:(I)V"); } private sealed class StubParser : ICallgraphParser diff --git a/src/Signals/__Tests/StellaOps.Signals.Tests/ReachabilityScoringServiceTests.cs b/src/Signals/__Tests/StellaOps.Signals.Tests/ReachabilityScoringServiceTests.cs index 718329987..1d0734a66 100644 --- a/src/Signals/__Tests/StellaOps.Signals.Tests/ReachabilityScoringServiceTests.cs +++ b/src/Signals/__Tests/StellaOps.Signals.Tests/ReachabilityScoringServiceTests.cs @@ -88,6 +88,79 @@ public class ReachabilityScoringServiceTests Assert.False(string.IsNullOrWhiteSpace(fact.Metadata?["fact.digest"])); } + [Fact] + public async Task RecomputeAsync_ComputesUncertaintyRiskScoreUsingConfiguredEntropyWeights() + { + var callgraph = new CallgraphDocument + { + Id = "cg-2", + Language = "java", + Component = "demo", + Version = "1.0.0", + Nodes = new List + { + new("main", "Main", "method", null, null, null), + new("svc", "Svc", "method", null, null, null), + new("target", "Target", "method", null, null, null) + }, + Edges = new List + { + new("main", "svc", "call"), + new("svc", "target", "call") + } + }; + + var callgraphRepository = new InMemoryCallgraphRepository(callgraph); + var factRepository = new InMemoryReachabilityFactRepository(); + + var options = new SignalsOptions(); + options.Scoring.ReachableConfidence = 0.8; + options.Scoring.RuntimeBonus = 0.1; + options.Scoring.UncertaintyEntropyMultiplier = 1.0; + options.Scoring.UncertaintyBoostCeiling = 0.3; + + var cache = new InMemoryReachabilityCache(); + var eventsPublisher = new RecordingEventsPublisher(); + var unknowns = new InMemoryUnknownsRepository(); + unknowns.Stored.Add(new UnknownSymbolDocument + { + SubjectKey = "demo|1.0.0", + CallgraphId = callgraph.Id, + SymbolId = "sym:unknown", + Reason = "fixture", + CreatedAt = DateTimeOffset.Parse("2025-12-13T00:00:00Z") + }); + + var service = new ReachabilityScoringService( + callgraphRepository, + factRepository, + TimeProvider.System, + Options.Create(options), + cache, + unknowns, + eventsPublisher, + NullLogger.Instance); + + var request = new ReachabilityRecomputeRequest + { + CallgraphId = callgraph.Id, + Subject = new ReachabilitySubject { Component = "demo", Version = "1.0.0" }, + EntryPoints = new List { "main" }, + Targets = new List { "target" }, + RuntimeHits = new List { "svc", "target" } + }; + + var fact = await service.RecomputeAsync(request, CancellationToken.None); + + Assert.NotNull(fact.Uncertainty); + Assert.Equal("T2", fact.Uncertainty!.AggregateTier); + Assert.Contains(fact.Uncertainty.States, state => state.Code == "U1"); + + Assert.Equal(0.26325, fact.Score, 5); // 0.405 base score * (1 - 0.35 unknowns penalty ceiling) + Assert.Equal(0.62775, fact.RiskScore, 5); // 0.405 base score * (1 + 0.25 tier modifier + 0.3 entropy boost ceiling) + Assert.Equal(fact.RiskScore, fact.Uncertainty.RiskScore ?? 0.0, 5); + } + private sealed class InMemoryCallgraphRepository : ICallgraphRepository { private readonly CallgraphDocument document; diff --git a/src/Telemetry/StellaOps.Telemetry.Core/StellaOps.Telemetry.Core.Tests/StellaOps.Telemetry.Core.Tests.csproj b/src/Telemetry/StellaOps.Telemetry.Core/StellaOps.Telemetry.Core.Tests/StellaOps.Telemetry.Core.Tests.csproj index 3cd9ba794..15f1679fa 100644 --- a/src/Telemetry/StellaOps.Telemetry.Core/StellaOps.Telemetry.Core.Tests/StellaOps.Telemetry.Core.Tests.csproj +++ b/src/Telemetry/StellaOps.Telemetry.Core/StellaOps.Telemetry.Core.Tests/StellaOps.Telemetry.Core.Tests.csproj @@ -12,7 +12,6 @@ - diff --git a/src/Web/StellaOps.Web/src/app/features/graph/graph-explorer.component.html b/src/Web/StellaOps.Web/src/app/features/graph/graph-explorer.component.html index 33209a35a..2b80f5f3a 100644 --- a/src/Web/StellaOps.Web/src/app/features/graph/graph-explorer.component.html +++ b/src/Web/StellaOps.Web/src/app/features/graph/graph-explorer.component.html @@ -80,14 +80,15 @@ - -
- - diff --git a/src/Web/StellaOps.Web/tests/e2e/a11y-smoke.spec.ts b/src/Web/StellaOps.Web/tests/e2e/a11y-smoke.spec.ts index 1e4292942..62d1305f9 100644 --- a/src/Web/StellaOps.Web/tests/e2e/a11y-smoke.spec.ts +++ b/src/Web/StellaOps.Web/tests/e2e/a11y-smoke.spec.ts @@ -92,7 +92,7 @@ test.describe('a11y-smoke', () => { await page.goto('/triage/artifacts/asset-web-prod'); await expect(page.getByRole('heading', { name: 'Artifact triage' })).toBeVisible({ timeout: 10000 }); - await page.getByRole('button', { name: 'VEX' }).first().click(); + await page.getByRole('button', { name: /^VEX$/ }).first().click(); await expect(page.getByRole('dialog', { name: 'VEX decision' })).toBeVisible({ timeout: 10000 }); const results = await new AxeBuilder({ page }) diff --git a/src/__Libraries/StellaOps.Auth.Security/Dpop/MessagingDpopReplayCache.cs b/src/__Libraries/StellaOps.Auth.Security/Dpop/MessagingDpopReplayCache.cs new file mode 100644 index 000000000..2512aa19d --- /dev/null +++ b/src/__Libraries/StellaOps.Auth.Security/Dpop/MessagingDpopReplayCache.cs @@ -0,0 +1,43 @@ +using StellaOps.Messaging.Abstractions; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// DPoP replay cache backed by . +/// Supports any transport (InMemory, Valkey, PostgreSQL) via factory injection. +/// +public sealed class MessagingDpopReplayCache : IDpopReplayCache +{ + private readonly IIdempotencyStore _store; + private readonly TimeProvider _timeProvider; + + public MessagingDpopReplayCache( + IIdempotencyStoreFactory storeFactory, + TimeProvider? timeProvider = null) + { + ArgumentNullException.ThrowIfNull(storeFactory); + + _store = storeFactory.Create("dpop:replay"); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask TryStoreAsync( + string jwtId, + DateTimeOffset expiresAt, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(jwtId); + + var now = _timeProvider.GetUtcNow(); + var ttl = expiresAt - now; + + if (ttl <= TimeSpan.Zero) + { + // Already expired, treat as valid to store (won't conflict) + return true; + } + + var result = await _store.TryClaimAsync(jwtId, jwtId, ttl, cancellationToken).ConfigureAwait(false); + return result.IsFirstClaim; + } +} diff --git a/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestTests.cs b/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestTests.cs index f2d979f2b..6a7834d8e 100644 --- a/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestTests.cs +++ b/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestTests.cs @@ -5,7 +5,7 @@ using Xunit; public class ReplayManifestTests { [Fact] - public void SerializesWithNamespacesAndAnalysis() + public void SerializesWithNamespacesAndAnalysis_V1() { var manifest = new ReplayManifest { @@ -20,7 +20,9 @@ public class ReplayManifestTests { Kind = "static", CasUri = "cas://reachability_graphs/aa/aagraph.tar.zst", - Sha256 = "aa", + Hash = "sha256:aa", + HashAlgorithm = "sha256", + Sha256 = "aa", // Legacy field for v1 compat Namespace = "reachability_graphs", CallgraphId = "cg-1", Analyzer = "scanner", @@ -31,7 +33,9 @@ public class ReplayManifestTests { Source = "runtime", CasUri = "cas://runtime_traces/bb/bbtrace.tar.zst", - Sha256 = "bb", + Hash = "sha256:bb", + HashAlgorithm = "sha256", + Sha256 = "bb", // Legacy field for v1 compat Namespace = "runtime_traces", RecordedAt = System.DateTimeOffset.Parse("2025-11-26T00:00:00Z") }); @@ -43,4 +47,36 @@ public class ReplayManifestTests Assert.Contains("\"callgraphId\":\"cg-1\"", json); Assert.Contains("\"namespace\":\"runtime_traces\"", json); } + + [Fact] + public void SerializesWithV2HashFields() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Reachability = new ReplayReachabilitySection + { + AnalysisId = "analysis-v2" + } + }; + + manifest.AddReachabilityGraph(new ReplayReachabilityGraphReference + { + Kind = "static", + CasUri = "cas://reachability/graphs/blake3:abc123", + Hash = "blake3:abc123def456789012345678901234567890123456789012345678901234", + HashAlgorithm = "blake3-256", + Namespace = "reachability_graphs", + Analyzer = "scanner.java@10.0.0", + Version = "10.0.0" + }); + + var json = JsonSerializer.Serialize(manifest, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + + Assert.Contains("\"schemaVersion\":\"2.0\"", json); + Assert.Contains("\"hash\":\"blake3:", json); + Assert.Contains("\"hashAlg\":\"blake3-256\"", json); + // v2 manifests should not emit legacy sha256 field (JsonIgnore when null) + Assert.DoesNotContain("\"sha256\":", json); + } } diff --git a/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestV2Tests.cs b/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestV2Tests.cs new file mode 100644 index 000000000..76d290853 --- /dev/null +++ b/src/__Libraries/StellaOps.Replay.Core.Tests/ReplayManifestV2Tests.cs @@ -0,0 +1,483 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Tasks; +using StellaOps.Replay.Core; +using Xunit; + +namespace StellaOps.Replay.Core.Tests; + +/// +/// Test vectors from replay-manifest-v2-acceptance.md +/// +public class ReplayManifestV2Tests +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false + }; + + #region Section 4.1: Minimal Valid Manifest v2 + + [Fact] + public void MinimalValidManifestV2_SerializesCorrectly() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Scan = new ReplayScanMetadata + { + Id = "scan-test-001", + Time = DateTimeOffset.Parse("2025-12-13T10:00:00Z") + }, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Kind = "static", + Analyzer = "scanner.java@10.2.0", + Hash = "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2" + } + }, + RuntimeTraces = new List(), + CodeIdCoverage = new CodeIdCoverage + { + TotalNodes = 100, + NodesWithSymbolId = 100, + NodesWithCodeId = 0, + CoveragePercent = 100.0 + } + } + }; + + var json = JsonSerializer.Serialize(manifest, JsonOptions); + + Assert.Contains("\"schemaVersion\":\"2.0\"", json); + Assert.Contains("\"hash\":\"blake3:", json); + Assert.Contains("\"hashAlg\":\"blake3-256\"", json); + Assert.Contains("\"code_id_coverage\"", json); + Assert.Contains("\"total_nodes\":100", json); + } + + #endregion + + #region Section 4.2: Manifest with Runtime Traces + + [Fact] + public void ManifestWithRuntimeTraces_SerializesCorrectly() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Scan = new ReplayScanMetadata + { + Id = "scan-test-002", + Time = DateTimeOffset.Parse("2025-12-13T11:00:00Z") + }, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Kind = "static", + Analyzer = "scanner.java@10.2.0", + Hash = "blake3:1111111111111111111111111111111111111111111111111111111111111111", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:1111111111111111111111111111111111111111111111111111111111111111" + } + }, + RuntimeTraces = new List + { + new() + { + Source = "eventpipe", + Hash = "sha256:2222222222222222222222222222222222222222222222222222222222222222", + HashAlgorithm = "sha256", + CasUri = "cas://reachability/runtime/sha256:2222222222222222222222222222222222222222222222222222222222222222", + RecordedAt = DateTimeOffset.Parse("2025-12-13T10:30:00Z") + } + } + } + }; + + var json = JsonSerializer.Serialize(manifest, JsonOptions); + + Assert.Contains("\"source\":\"eventpipe\"", json); + Assert.Contains("\"hash\":\"sha256:", json); + Assert.Contains("\"hashAlg\":\"sha256\"", json); + } + + #endregion + + #region Section 4.3: Sorting Validation + + [Fact] + public void SortingValidation_UnsortedGraphs_FailsValidation() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Kind = "framework", + Hash = "blake3:zzzz1111111111111111111111111111111111111111111111111111111111", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:zzzz..." + }, + new() + { + Kind = "static", + Hash = "blake3:aaaa1111111111111111111111111111111111111111111111111111111111", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:aaaa..." + } + } + } + }; + + var validator = new ReplayManifestValidator(); + var result = validator.ValidateAsync(manifest).GetAwaiter().GetResult(); + + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.ErrorCode == ReplayManifestErrorCodes.UnsortedEntries); + } + + [Fact] + public void SortingValidation_SortedGraphs_PassesValidation() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Kind = "static", + Hash = "blake3:aaaa1111111111111111111111111111111111111111111111111111111111", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:aaaa..." + }, + new() + { + Kind = "framework", + Hash = "blake3:zzzz1111111111111111111111111111111111111111111111111111111111", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:zzzz..." + } + } + } + }; + + var validator = new ReplayManifestValidator(); + var result = validator.ValidateAsync(manifest).GetAwaiter().GetResult(); + + Assert.True(result.IsValid); + } + + #endregion + + #region Section 4.4: Invalid Manifest Vectors + + [Fact] + public void InvalidManifest_MissingSchemaVersion_FailsValidation() + { + var manifest = new ReplayManifest + { + SchemaVersion = null! + }; + + var validator = new ReplayManifestValidator(); + var result = validator.ValidateAsync(manifest).GetAwaiter().GetResult(); + + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.ErrorCode == ReplayManifestErrorCodes.MissingVersion); + } + + [Fact] + public void InvalidManifest_VersionMismatch_WhenV2Required_FailsValidation() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V1 + }; + + var validator = new ReplayManifestValidator(requireV2: true); + var result = validator.ValidateAsync(manifest).GetAwaiter().GetResult(); + + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.ErrorCode == ReplayManifestErrorCodes.VersionMismatch); + } + + [Fact] + public void InvalidManifest_MissingHashAlg_InV2_FailsValidation() + { + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Hash = "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + HashAlgorithm = null!, // Missing + CasUri = "cas://reachability/graphs/blake3:..." + } + } + } + }; + + var validator = new ReplayManifestValidator(); + var result = validator.ValidateAsync(manifest).GetAwaiter().GetResult(); + + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.ErrorCode == ReplayManifestErrorCodes.MissingHashAlg); + } + + [Fact] + public async Task InvalidManifest_MissingCasReference_FailsValidation() + { + var casValidator = new InMemoryCasValidator(); + // Don't register any objects + + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Hash = "blake3:a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:missing" + } + } + } + }; + + var validator = new ReplayManifestValidator(casValidator); + var result = await validator.ValidateAsync(manifest); + + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.ErrorCode == ReplayManifestErrorCodes.CasNotFound); + } + + [Fact] + public async Task InvalidManifest_HashMismatch_FailsValidation() + { + var casValidator = new InMemoryCasValidator(); + casValidator.Register( + "cas://reachability/graphs/blake3:actual", + "blake3:differenthash"); + casValidator.Register( + "cas://reachability/graphs/blake3:actual.dsse", + "blake3:differenthash.dsse"); + + var manifest = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Hash = "blake3:expected", + HashAlgorithm = "blake3-256", + CasUri = "cas://reachability/graphs/blake3:actual" + } + } + } + }; + + var validator = new ReplayManifestValidator(casValidator); + var result = await validator.ValidateAsync(manifest); + + Assert.False(result.IsValid); + Assert.Contains(result.Errors, e => e.ErrorCode == ReplayManifestErrorCodes.HashMismatch); + } + + #endregion + + #region Section 5: Migration Path + + [Fact] + public void UpgradeToV2_ConvertsV1ManifestCorrectly() + { + var v1 = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V1, + Scan = new ReplayScanMetadata + { + Id = "scan-legacy" + }, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() + { + Kind = "static", + Sha256 = "abc123", + CasUri = "cas://reachability/graphs/abc123" + } + } + } + }; + + var v2 = ReplayManifestValidator.UpgradeToV2(v1); + + Assert.Equal(ReplayManifestVersions.V2, v2.SchemaVersion); + Assert.Single(v2.Reachability.Graphs); + Assert.Equal("sha256:abc123", v2.Reachability.Graphs[0].Hash); + Assert.Equal("sha256", v2.Reachability.Graphs[0].HashAlgorithm); + } + + [Fact] + public void UpgradeToV2_SortsGraphsByUri() + { + var v1 = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V1, + Reachability = new ReplayReachabilitySection + { + Graphs = new List + { + new() { Sha256 = "zzz", CasUri = "cas://graphs/zzz" }, + new() { Sha256 = "aaa", CasUri = "cas://graphs/aaa" } + } + } + }; + + var v2 = ReplayManifestValidator.UpgradeToV2(v1); + + Assert.Equal("cas://graphs/aaa", v2.Reachability.Graphs[0].CasUri); + Assert.Equal("cas://graphs/zzz", v2.Reachability.Graphs[1].CasUri); + } + + #endregion + + #region ReachabilityReplayWriter Tests + + [Fact] + public void BuildManifestV2_WithValidGraphs_CreatesSortedManifest() + { + var scan = new ReplayScanMetadata { Id = "test-scan" }; + var graphs = new[] + { + new ReplayReachabilityGraphReference + { + Hash = "blake3:zzzz", + CasUri = "cas://graphs/zzzz" + }, + new ReplayReachabilityGraphReference + { + Hash = "blake3:aaaa", + CasUri = "cas://graphs/aaaa" + } + }; + + var manifest = ReachabilityReplayWriter.BuildManifestV2( + scan, + graphs, + Array.Empty()); + + Assert.Equal(ReplayManifestVersions.V2, manifest.SchemaVersion); + Assert.Equal("cas://graphs/aaaa", manifest.Reachability.Graphs[0].CasUri); + Assert.Equal("cas://graphs/zzzz", manifest.Reachability.Graphs[1].CasUri); + } + + [Fact] + public void BuildManifestV2_WithLegacySha256_MigratesHashField() + { + var scan = new ReplayScanMetadata { Id = "test-scan" }; + var graphs = new[] + { + new ReplayReachabilityGraphReference + { + Sha256 = "abc123", + CasUri = "cas://graphs/abc123" + } + }; + + var manifest = ReachabilityReplayWriter.BuildManifestV2( + scan, + graphs, + Array.Empty()); + + Assert.Equal("sha256:abc123", manifest.Reachability.Graphs[0].Hash); + Assert.Equal("sha256", manifest.Reachability.Graphs[0].HashAlgorithm); + } + + [Fact] + public void BuildManifestV2_InfersHashAlgorithmFromPrefix() + { + var scan = new ReplayScanMetadata { Id = "test-scan" }; + var graphs = new[] + { + new ReplayReachabilityGraphReference + { + Hash = "blake3:a1b2c3d4", + CasUri = "cas://graphs/a1b2c3d4" + } + }; + + var manifest = ReachabilityReplayWriter.BuildManifestV2( + scan, + graphs, + Array.Empty()); + + Assert.Equal("blake3-256", manifest.Reachability.Graphs[0].HashAlgorithm); + } + + [Fact] + public void BuildManifestV2_RequiresAtLeastOneGraph() + { + var scan = new ReplayScanMetadata { Id = "test-scan" }; + + Assert.Throws(() => + ReachabilityReplayWriter.BuildManifestV2( + scan, + Array.Empty(), + Array.Empty())); + } + + #endregion + + #region CodeIdCoverage Tests + + [Fact] + public void CodeIdCoverage_SerializesWithSnakeCaseKeys() + { + var coverage = new CodeIdCoverage + { + TotalNodes = 1247, + NodesWithSymbolId = 1189, + NodesWithCodeId = 58, + CoveragePercent = 100.0 + }; + + var json = JsonSerializer.Serialize(coverage, JsonOptions); + + Assert.Contains("\"total_nodes\":1247", json); + Assert.Contains("\"nodes_with_symbol_id\":1189", json); + Assert.Contains("\"nodes_with_code_id\":58", json); + Assert.Contains("\"coverage_percent\":100", json); + } + + #endregion +} diff --git a/src/__Libraries/StellaOps.Replay.Core/CasValidator.cs b/src/__Libraries/StellaOps.Replay.Core/CasValidator.cs new file mode 100644 index 000000000..3315cf69c --- /dev/null +++ b/src/__Libraries/StellaOps.Replay.Core/CasValidator.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace StellaOps.Replay.Core; + +/// +/// Validates CAS references before manifest signing. +/// +public interface ICasValidator +{ + /// + /// Validates that a CAS URI exists and matches the expected hash. + /// + Task ValidateAsync(string casUri, string expectedHash); + + /// + /// Validates multiple CAS references in batch. + /// + Task ValidateBatchAsync(IEnumerable references); +} + +/// +/// A reference to a CAS object for validation. +/// +public sealed record CasReference( + string CasUri, + string ExpectedHash, + string? HashAlgorithm = null +); + +/// +/// Result of a CAS validation operation. +/// +public sealed record CasValidationResult( + bool IsValid, + string? ActualHash = null, + string? Error = null, + IReadOnlyList? Errors = null +) +{ + public static CasValidationResult Success(string actualHash) => + new(true, actualHash); + + public static CasValidationResult Failure(string error) => + new(false, Error: error); + + public static CasValidationResult NotFound(string casUri) => + new(false, Error: $"CAS object not found: {casUri}"); + + public static CasValidationResult HashMismatch(string casUri, string expected, string actual) => + new(false, ActualHash: actual, Error: $"Hash mismatch for {casUri}: expected {expected}, got {actual}"); + + public static CasValidationResult BatchResult(bool isValid, IReadOnlyList errors) => + new(isValid, Errors: errors); +} + +/// +/// Error details for a single CAS validation failure in a batch. +/// +public sealed record CasValidationError( + string CasUri, + string ErrorCode, + string Message +); + +/// +/// In-memory CAS validator for testing and offline scenarios. +/// +public sealed class InMemoryCasValidator : ICasValidator +{ + private readonly Dictionary _objects = new(StringComparer.Ordinal); + + /// + /// Registers a CAS object for validation. + /// + public void Register(string casUri, string hash) + { + _objects[casUri] = hash; + } + + public Task ValidateAsync(string casUri, string expectedHash) + { + if (!_objects.TryGetValue(casUri, out var actualHash)) + { + return Task.FromResult(CasValidationResult.NotFound(casUri)); + } + + if (!string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase)) + { + return Task.FromResult(CasValidationResult.HashMismatch(casUri, expectedHash, actualHash)); + } + + return Task.FromResult(CasValidationResult.Success(actualHash)); + } + + public async Task ValidateBatchAsync(IEnumerable references) + { + var errors = new List(); + foreach (var reference in references) + { + var result = await ValidateAsync(reference.CasUri, reference.ExpectedHash).ConfigureAwait(false); + if (!result.IsValid) + { + errors.Add(new CasValidationError( + reference.CasUri, + result.Error?.Contains("not found") == true + ? ReplayManifestErrorCodes.CasNotFound + : ReplayManifestErrorCodes.HashMismatch, + result.Error ?? "Unknown error" + )); + } + } + + return CasValidationResult.BatchResult(errors.Count == 0, errors); + } +} diff --git a/src/__Libraries/StellaOps.Replay.Core/ReachabilityReplayWriter.cs b/src/__Libraries/StellaOps.Replay.Core/ReachabilityReplayWriter.cs index 2e5695f99..101dec0a6 100644 --- a/src/__Libraries/StellaOps.Replay.Core/ReachabilityReplayWriter.cs +++ b/src/__Libraries/StellaOps.Replay.Core/ReachabilityReplayWriter.cs @@ -58,17 +58,43 @@ public static class ReachabilityReplayWriter throw new InvalidOperationException("Graph casUri is required."); } - if (string.IsNullOrWhiteSpace(graph.Sha256)) + // v2: Prefer Hash field with algorithm prefix + if (string.IsNullOrWhiteSpace(graph.Hash)) { - throw new InvalidOperationException("Graph sha256 is required."); + // Backward compat: migrate from legacy Sha256 field + if (!string.IsNullOrWhiteSpace(graph.Sha256)) + { + graph.Hash = $"sha256:{graph.Sha256}"; + graph.HashAlgorithm = "sha256"; + } + else + { + throw new InvalidOperationException("Graph hash is required."); + } + } + + // Normalize hash algorithm from hash prefix if not explicitly set + if (string.IsNullOrWhiteSpace(graph.HashAlgorithm)) + { + graph.HashAlgorithm = InferHashAlgorithm(graph.Hash); } - graph.HashAlgorithm = string.IsNullOrWhiteSpace(graph.HashAlgorithm) ? "blake3-256" : graph.HashAlgorithm; graph.Kind = string.IsNullOrWhiteSpace(graph.Kind) ? "static" : graph.Kind; graph.Namespace = string.IsNullOrWhiteSpace(graph.Namespace) ? "reachability_graphs" : graph.Namespace; return graph; } + private static string InferHashAlgorithm(string hash) + { + if (hash.StartsWith("blake3:", StringComparison.OrdinalIgnoreCase)) + return "blake3-256"; + if (hash.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + return "sha256"; + if (hash.StartsWith("sha512:", StringComparison.OrdinalIgnoreCase)) + return "sha512"; + return "blake3-256"; // Default for v2 + } + private static ReplayReachabilityTraceReference NormalizeTrace(ReplayReachabilityTraceReference trace) { if (string.IsNullOrWhiteSpace(trace.CasUri)) @@ -76,12 +102,27 @@ public static class ReachabilityReplayWriter throw new InvalidOperationException("Trace casUri is required."); } - if (string.IsNullOrWhiteSpace(trace.Sha256)) + // v2: Prefer Hash field with algorithm prefix + if (string.IsNullOrWhiteSpace(trace.Hash)) { - throw new InvalidOperationException("Trace sha256 is required."); + // Backward compat: migrate from legacy Sha256 field + if (!string.IsNullOrWhiteSpace(trace.Sha256)) + { + trace.Hash = $"sha256:{trace.Sha256}"; + trace.HashAlgorithm = "sha256"; + } + else + { + throw new InvalidOperationException("Trace hash is required."); + } + } + + // Normalize hash algorithm from hash prefix if not explicitly set + if (string.IsNullOrWhiteSpace(trace.HashAlgorithm)) + { + trace.HashAlgorithm = InferHashAlgorithm(trace.Hash); } - trace.HashAlgorithm = string.IsNullOrWhiteSpace(trace.HashAlgorithm) ? "sha256" : trace.HashAlgorithm; trace.Namespace = string.IsNullOrWhiteSpace(trace.Namespace) ? "runtime_traces" : trace.Namespace; trace.Source = string.IsNullOrWhiteSpace(trace.Source) ? "runtime" : trace.Source; return trace; diff --git a/src/__Libraries/StellaOps.Replay.Core/ReplayManifest.cs b/src/__Libraries/StellaOps.Replay.Core/ReplayManifest.cs index 8688df6a9..4681ca08e 100644 --- a/src/__Libraries/StellaOps.Replay.Core/ReplayManifest.cs +++ b/src/__Libraries/StellaOps.Replay.Core/ReplayManifest.cs @@ -47,6 +47,24 @@ public sealed class ReplayReachabilitySection [JsonPropertyName("runtimeTraces")] public List RuntimeTraces { get; set; } = new(); + + [JsonPropertyName("code_id_coverage")] + public CodeIdCoverage? CodeIdCoverage { get; set; } +} + +public sealed class CodeIdCoverage +{ + [JsonPropertyName("total_nodes")] + public int TotalNodes { get; set; } + + [JsonPropertyName("nodes_with_symbol_id")] + public int NodesWithSymbolId { get; set; } + + [JsonPropertyName("nodes_with_code_id")] + public int NodesWithCodeId { get; set; } + + [JsonPropertyName("coverage_percent")] + public double CoveragePercent { get; set; } } public sealed class ReplayReachabilityGraphReference @@ -57,11 +75,22 @@ public sealed class ReplayReachabilityGraphReference [JsonPropertyName("casUri")] public string CasUri { get; set; } = string.Empty; - [JsonPropertyName("sha256")] - public string Sha256 { get; set; } = string.Empty; + /// + /// Hash with algorithm prefix, e.g., "blake3:a1b2c3d4..." or "sha256:feedface..." + /// + [JsonPropertyName("hash")] + public string Hash { get; set; } = string.Empty; [JsonPropertyName("hashAlg")] - public string HashAlgorithm { get; set; } = "sha256"; + public string HashAlgorithm { get; set; } = "blake3-256"; + + /// + /// Legacy SHA-256 field for backward compatibility with v1 manifests. + /// In v2, use the Hash field with algorithm prefix instead. + /// + [JsonPropertyName("sha256")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Sha256 { get; set; } [JsonPropertyName("namespace")] public string Namespace { get; set; } = "reachability_graphs"; @@ -84,12 +113,23 @@ public sealed class ReplayReachabilityTraceReference [JsonPropertyName("casUri")] public string CasUri { get; set; } = string.Empty; - [JsonPropertyName("sha256")] - public string Sha256 { get; set; } = string.Empty; + /// + /// Hash with algorithm prefix, e.g., "sha256:feedface..." + /// + [JsonPropertyName("hash")] + public string Hash { get; set; } = string.Empty; [JsonPropertyName("hashAlg")] public string HashAlgorithm { get; set; } = "sha256"; + /// + /// Legacy SHA-256 field for backward compatibility with v1 manifests. + /// In v2, use the Hash field with algorithm prefix instead. + /// + [JsonPropertyName("sha256")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Sha256 { get; set; } + [JsonPropertyName("namespace")] public string Namespace { get; set; } = "runtime_traces"; diff --git a/src/__Libraries/StellaOps.Replay.Core/ReplayManifestValidator.cs b/src/__Libraries/StellaOps.Replay.Core/ReplayManifestValidator.cs new file mode 100644 index 000000000..863a8a1e2 --- /dev/null +++ b/src/__Libraries/StellaOps.Replay.Core/ReplayManifestValidator.cs @@ -0,0 +1,397 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace StellaOps.Replay.Core; + +/// +/// Error codes for replay manifest validation per acceptance contract. +/// +public static class ReplayManifestErrorCodes +{ + public const string MissingVersion = "REPLAY_MANIFEST_MISSING_VERSION"; + public const string VersionMismatch = "REPLAY_MANIFEST_VERSION_MISMATCH"; + public const string MissingHashAlg = "REPLAY_MANIFEST_MISSING_HASH_ALG"; + public const string UnsortedEntries = "REPLAY_MANIFEST_UNSORTED_ENTRIES"; + public const string CasNotFound = "REPLAY_MANIFEST_CAS_NOT_FOUND"; + public const string HashMismatch = "REPLAY_MANIFEST_HASH_MISMATCH"; + public const string MissingHash = "REPLAY_MANIFEST_MISSING_HASH"; + public const string MissingCasUri = "REPLAY_MANIFEST_MISSING_CAS_URI"; + public const string InvalidHashFormat = "REPLAY_MANIFEST_INVALID_HASH_FORMAT"; +} + +/// +/// Result of manifest validation. +/// +public sealed record ManifestValidationResult( + bool IsValid, + IReadOnlyList Errors +) +{ + public static ManifestValidationResult Success() => + new(true, Array.Empty()); + + public static ManifestValidationResult Failure(IEnumerable errors) => + new(false, errors.ToList()); + + public static ManifestValidationResult Failure(ManifestValidationError error) => + new(false, new[] { error }); +} + +/// +/// A single validation error. +/// +public sealed record ManifestValidationError( + string ErrorCode, + string Message, + string? Path = null +); + +/// +/// Validates replay manifests against v2 schema rules and CAS registration requirements. +/// +public sealed class ReplayManifestValidator +{ + private readonly ICasValidator? _casValidator; + private readonly bool _requireV2; + + /// + /// Creates a validator with optional CAS validation. + /// + /// Optional CAS validator for reference verification. + /// If true, only v2 manifests are accepted. + public ReplayManifestValidator(ICasValidator? casValidator = null, bool requireV2 = false) + { + _casValidator = casValidator; + _requireV2 = requireV2; + } + + /// + /// Validates a manifest against v2 schema rules. + /// + public async Task ValidateAsync(ReplayManifest manifest) + { + ArgumentNullException.ThrowIfNull(manifest); + var errors = new List(); + + // 1. Validate schema version + if (string.IsNullOrWhiteSpace(manifest.SchemaVersion)) + { + errors.Add(new ManifestValidationError( + ReplayManifestErrorCodes.MissingVersion, + "schemaVersion is required", + "schemaVersion")); + } + else if (_requireV2 && manifest.SchemaVersion != ReplayManifestVersions.V2) + { + errors.Add(new ManifestValidationError( + ReplayManifestErrorCodes.VersionMismatch, + $"schemaVersion must be {ReplayManifestVersions.V2} when v2 is required", + "schemaVersion")); + } + + // 2. Validate graph references + var isV2 = manifest.SchemaVersion == ReplayManifestVersions.V2; + var graphErrors = ValidateGraphs(manifest.Reachability?.Graphs, isV2); + errors.AddRange(graphErrors); + + // 3. Validate trace references + var traceErrors = ValidateTraces(manifest.Reachability?.RuntimeTraces, isV2); + errors.AddRange(traceErrors); + + // 4. Validate sorting (v2 only) + if (isV2) + { + var sortingErrors = ValidateSorting(manifest); + errors.AddRange(sortingErrors); + } + + // 5. Validate CAS registration if validator provided + if (_casValidator is not null && errors.Count == 0) + { + var casErrors = await ValidateCasReferencesAsync(manifest).ConfigureAwait(false); + errors.AddRange(casErrors); + } + + return errors.Count == 0 + ? ManifestValidationResult.Success() + : ManifestValidationResult.Failure(errors); + } + + private static IEnumerable ValidateGraphs( + IReadOnlyList? graphs, bool isV2) + { + if (graphs is null || graphs.Count == 0) + yield break; + + for (var i = 0; i < graphs.Count; i++) + { + var graph = graphs[i]; + var path = $"reachability.graphs[{i}]"; + + if (string.IsNullOrWhiteSpace(graph.CasUri)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.MissingCasUri, + "casUri is required", + $"{path}.casUri"); + } + + if (isV2) + { + // v2 requires hash field with algorithm prefix + if (string.IsNullOrWhiteSpace(graph.Hash)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.MissingHash, + "hash is required in v2", + $"{path}.hash"); + } + else if (!graph.Hash.Contains(':')) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.InvalidHashFormat, + "hash must include algorithm prefix (e.g., blake3:...)", + $"{path}.hash"); + } + + if (string.IsNullOrWhiteSpace(graph.HashAlgorithm)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.MissingHashAlg, + "hashAlg is required in v2", + $"{path}.hashAlg"); + } + } + } + } + + private static IEnumerable ValidateTraces( + IReadOnlyList? traces, bool isV2) + { + if (traces is null || traces.Count == 0) + yield break; + + for (var i = 0; i < traces.Count; i++) + { + var trace = traces[i]; + var path = $"reachability.runtimeTraces[{i}]"; + + if (string.IsNullOrWhiteSpace(trace.CasUri)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.MissingCasUri, + "casUri is required", + $"{path}.casUri"); + } + + if (isV2) + { + // v2 requires hash field with algorithm prefix + if (string.IsNullOrWhiteSpace(trace.Hash)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.MissingHash, + "hash is required in v2", + $"{path}.hash"); + } + else if (!trace.Hash.Contains(':')) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.InvalidHashFormat, + "hash must include algorithm prefix (e.g., sha256:...)", + $"{path}.hash"); + } + + if (string.IsNullOrWhiteSpace(trace.HashAlgorithm)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.MissingHashAlg, + "hashAlg is required in v2", + $"{path}.hashAlg"); + } + } + } + } + + private static IEnumerable ValidateSorting(ReplayManifest manifest) + { + var graphs = manifest.Reachability?.Graphs; + if (graphs is not null && graphs.Count > 1) + { + var sorted = graphs.OrderBy(g => g.CasUri, StringComparer.Ordinal).ToList(); + for (var i = 0; i < graphs.Count; i++) + { + if (!string.Equals(graphs[i].CasUri, sorted[i].CasUri, StringComparison.Ordinal)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.UnsortedEntries, + "reachability.graphs must be sorted by casUri (lexicographic)", + "reachability.graphs"); + break; + } + } + } + + var traces = manifest.Reachability?.RuntimeTraces; + if (traces is not null && traces.Count > 1) + { + var sorted = traces.OrderBy(t => t.CasUri, StringComparer.Ordinal).ToList(); + for (var i = 0; i < traces.Count; i++) + { + if (!string.Equals(traces[i].CasUri, sorted[i].CasUri, StringComparison.Ordinal)) + { + yield return new ManifestValidationError( + ReplayManifestErrorCodes.UnsortedEntries, + "reachability.runtimeTraces must be sorted by casUri (lexicographic)", + "reachability.runtimeTraces"); + break; + } + } + } + } + + private async Task> ValidateCasReferencesAsync(ReplayManifest manifest) + { + var references = new List(); + + // Collect graph references + if (manifest.Reachability?.Graphs is not null) + { + foreach (var graph in manifest.Reachability.Graphs) + { + if (!string.IsNullOrWhiteSpace(graph.CasUri) && !string.IsNullOrWhiteSpace(graph.Hash)) + { + references.Add(new CasReference(graph.CasUri, graph.Hash, graph.HashAlgorithm)); + + // Also check for DSSE envelope + var dsseUri = $"{graph.CasUri}.dsse"; + references.Add(new CasReference(dsseUri, $"{graph.Hash}.dsse", graph.HashAlgorithm)); + } + } + } + + // Collect trace references + if (manifest.Reachability?.RuntimeTraces is not null) + { + foreach (var trace in manifest.Reachability.RuntimeTraces) + { + if (!string.IsNullOrWhiteSpace(trace.CasUri) && !string.IsNullOrWhiteSpace(trace.Hash)) + { + references.Add(new CasReference(trace.CasUri, trace.Hash, trace.HashAlgorithm)); + } + } + } + + if (references.Count == 0) + return Array.Empty(); + + var result = await _casValidator!.ValidateBatchAsync(references).ConfigureAwait(false); + if (result.IsValid) + return Array.Empty(); + + return result.Errors?.Select(e => new ManifestValidationError(e.ErrorCode, e.Message, e.CasUri)) + ?? Array.Empty(); + } + + /// + /// Upgrades a v1 manifest to v2 format. + /// + public static ReplayManifest UpgradeToV2(ReplayManifest v1) + { + ArgumentNullException.ThrowIfNull(v1); + + var v2 = new ReplayManifest + { + SchemaVersion = ReplayManifestVersions.V2, + Scan = v1.Scan, + Reachability = new ReplayReachabilitySection + { + AnalysisId = v1.Reachability?.AnalysisId, + CodeIdCoverage = v1.Reachability?.CodeIdCoverage, + Graphs = v1.Reachability?.Graphs? + .Select(g => UpgradeGraph(g)) + .OrderBy(g => g.CasUri, StringComparer.Ordinal) + .ToList() ?? new List(), + RuntimeTraces = v1.Reachability?.RuntimeTraces? + .Select(t => UpgradeTrace(t)) + .OrderBy(t => t.CasUri, StringComparer.Ordinal) + .ToList() ?? new List() + } + }; + + return v2; + } + + private static ReplayReachabilityGraphReference UpgradeGraph(ReplayReachabilityGraphReference g) + { + var hash = g.Hash; + var hashAlg = g.HashAlgorithm; + + // If Hash is empty, derive from legacy Sha256 + if (string.IsNullOrWhiteSpace(hash) && !string.IsNullOrWhiteSpace(g.Sha256)) + { + hash = $"sha256:{g.Sha256}"; + hashAlg = "sha256"; + } + + // Infer hash algorithm from prefix if not set + if (string.IsNullOrWhiteSpace(hashAlg) && !string.IsNullOrWhiteSpace(hash)) + { + hashAlg = InferHashAlgorithmFromPrefix(hash); + } + + return new ReplayReachabilityGraphReference + { + Kind = g.Kind, + CasUri = g.CasUri, + Hash = hash ?? string.Empty, + HashAlgorithm = hashAlg ?? "blake3-256", + Namespace = g.Namespace, + CallgraphId = g.CallgraphId, + Analyzer = g.Analyzer, + Version = g.Version + }; + } + + private static ReplayReachabilityTraceReference UpgradeTrace(ReplayReachabilityTraceReference t) + { + var hash = t.Hash; + var hashAlg = t.HashAlgorithm; + + // If Hash is empty, derive from legacy Sha256 + if (string.IsNullOrWhiteSpace(hash) && !string.IsNullOrWhiteSpace(t.Sha256)) + { + hash = $"sha256:{t.Sha256}"; + hashAlg = "sha256"; + } + + // Infer hash algorithm from prefix if not set + if (string.IsNullOrWhiteSpace(hashAlg) && !string.IsNullOrWhiteSpace(hash)) + { + hashAlg = InferHashAlgorithmFromPrefix(hash); + } + + return new ReplayReachabilityTraceReference + { + Source = t.Source, + CasUri = t.CasUri, + Hash = hash ?? string.Empty, + HashAlgorithm = hashAlg ?? "sha256", + Namespace = t.Namespace, + RecordedAt = t.RecordedAt + }; + } + + private static string InferHashAlgorithmFromPrefix(string hash) + { + if (hash.StartsWith("blake3:", StringComparison.OrdinalIgnoreCase)) + return "blake3-256"; + if (hash.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + return "sha256"; + if (hash.StartsWith("sha512:", StringComparison.OrdinalIgnoreCase)) + return "sha512"; + return "blake3-256"; + } +} diff --git a/src/__Libraries/__Tests/StellaOps.Microservice.Tests/StellaOps.Microservice.Tests.csproj b/src/__Libraries/__Tests/StellaOps.Microservice.Tests/StellaOps.Microservice.Tests.csproj index 7468efb50..254d25c57 100644 --- a/src/__Libraries/__Tests/StellaOps.Microservice.Tests/StellaOps.Microservice.Tests.csproj +++ b/src/__Libraries/__Tests/StellaOps.Microservice.Tests/StellaOps.Microservice.Tests.csproj @@ -10,7 +10,7 @@ $(NoWarn);CA2255 false StellaOps.Microservice.Tests - + false diff --git a/src/__Libraries/__Tests/StellaOps.Router.Common.Tests/StellaOps.Router.Common.Tests.csproj b/src/__Libraries/__Tests/StellaOps.Router.Common.Tests/StellaOps.Router.Common.Tests.csproj index 7651d54ee..ccb600047 100644 --- a/src/__Libraries/__Tests/StellaOps.Router.Common.Tests/StellaOps.Router.Common.Tests.csproj +++ b/src/__Libraries/__Tests/StellaOps.Router.Common.Tests/StellaOps.Router.Common.Tests.csproj @@ -10,7 +10,7 @@ $(NoWarn);CA2255 false StellaOps.Router.Common.Tests - + false diff --git a/src/__Libraries/__Tests/StellaOps.Router.Config.Tests/StellaOps.Router.Config.Tests.csproj b/src/__Libraries/__Tests/StellaOps.Router.Config.Tests/StellaOps.Router.Config.Tests.csproj index 241baa850..de0a80f08 100644 --- a/src/__Libraries/__Tests/StellaOps.Router.Config.Tests/StellaOps.Router.Config.Tests.csproj +++ b/src/__Libraries/__Tests/StellaOps.Router.Config.Tests/StellaOps.Router.Config.Tests.csproj @@ -10,7 +10,7 @@ $(NoWarn);CA2255 false StellaOps.Router.Config.Tests - + false diff --git a/src/__Libraries/__Tests/StellaOps.Router.Transport.InMemory.Tests/StellaOps.Router.Transport.InMemory.Tests.csproj b/src/__Libraries/__Tests/StellaOps.Router.Transport.InMemory.Tests/StellaOps.Router.Transport.InMemory.Tests.csproj index 5a121a129..553bb3510 100644 --- a/src/__Libraries/__Tests/StellaOps.Router.Transport.InMemory.Tests/StellaOps.Router.Transport.InMemory.Tests.csproj +++ b/src/__Libraries/__Tests/StellaOps.Router.Transport.InMemory.Tests/StellaOps.Router.Transport.InMemory.Tests.csproj @@ -10,7 +10,7 @@ $(NoWarn);CA2255 false StellaOps.Router.Transport.InMemory.Tests - + false diff --git a/src/__Libraries/__Tests/StellaOps.Router.Transport.RabbitMq.Tests/StellaOps.Router.Transport.RabbitMq.Tests.csproj b/src/__Libraries/__Tests/StellaOps.Router.Transport.RabbitMq.Tests/StellaOps.Router.Transport.RabbitMq.Tests.csproj index fe3bc896c..58bf4ed01 100644 --- a/src/__Libraries/__Tests/StellaOps.Router.Transport.RabbitMq.Tests/StellaOps.Router.Transport.RabbitMq.Tests.csproj +++ b/src/__Libraries/__Tests/StellaOps.Router.Transport.RabbitMq.Tests/StellaOps.Router.Transport.RabbitMq.Tests.csproj @@ -10,7 +10,7 @@ $(NoWarn);CA2255 false StellaOps.Router.Transport.RabbitMq.Tests - + false diff --git a/tests/reachability/StellaOps.Reachability.FixtureTests/CorpusFixtureTests.cs b/tests/reachability/StellaOps.Reachability.FixtureTests/CorpusFixtureTests.cs index 791576526..4f3396f6d 100644 --- a/tests/reachability/StellaOps.Reachability.FixtureTests/CorpusFixtureTests.cs +++ b/tests/reachability/StellaOps.Reachability.FixtureTests/CorpusFixtureTests.cs @@ -1,6 +1,5 @@ using System.Security.Cryptography; using System.Text.Json; -using System.Text.RegularExpressions; using FluentAssertions; using Xunit; @@ -55,29 +54,47 @@ public class CorpusFixtureTests } [Fact] - public void ExpectFilesContainRequiredFields() + public void GroundTruthFilesContainRequiredFields() { var manifestPath = Path.Combine(CorpusRoot, "manifest.json"); var manifest = JsonDocument.Parse(File.ReadAllBytes(manifestPath)).RootElement.EnumerateArray().ToArray(); - var required = new[] { "id", "language", "state", "score" }; - var idRegex = new Regex(@"^id:\s*(?.+)$", RegexOptions.Multiline); + const string expectedSchemaVersion = "reachbench.reachgraph.truth/v1"; + var allowedVariants = new[] { "reachable", "unreachable" }; foreach (var entry in manifest) { var id = entry.GetProperty("id").GetString()!; var language = entry.GetProperty("language").GetString()!; - var expectPath = Path.Combine(CorpusRoot, language, id, "expect.yaml"); - File.Exists(expectPath).Should().BeTrue($"{id} missing expect.yaml"); - var text = File.ReadAllText(expectPath); + var truthPath = Path.Combine(CorpusRoot, language, id, "ground-truth.json"); + File.Exists(truthPath).Should().BeTrue($"{id} missing ground-truth.json"); - foreach (var field in required) + using var truthDoc = JsonDocument.Parse(File.ReadAllBytes(truthPath)); + truthDoc.RootElement.GetProperty("schema_version").GetString().Should().Be(expectedSchemaVersion, $"{id} ground-truth schema_version mismatch"); + truthDoc.RootElement.GetProperty("case_id").GetString().Should().Be(id, $"{id} ground-truth case_id must match manifest id"); + + var variant = truthDoc.RootElement.GetProperty("variant").GetString(); + variant.Should().NotBeNullOrWhiteSpace($"{id} ground-truth must set variant"); + allowedVariants.Should().Contain(variant!, $"{id} variant must be reachable|unreachable"); + + truthDoc.RootElement.TryGetProperty("paths", out var pathsProp).Should().BeTrue($"{id} ground-truth must include paths"); + pathsProp.ValueKind.Should().Be(JsonValueKind.Array, $"{id} paths must be an array"); + + if (string.Equals(variant, "reachable", StringComparison.Ordinal)) { - text.Should().Contain($"{field}:", $"{id} expect.yaml missing '{field}:'"); + pathsProp.GetArrayLength().Should().BeGreaterThan(0, $"{id} reachable ground-truth should include at least one path"); } - var match = idRegex.Match(text); - match.Success.Should().BeTrue($"{id} expect.yaml should include matching id"); - match.Groups["id"].Value.Trim().Should().Be(id, $"{id} expect.yaml id must match manifest id"); + foreach (var path in pathsProp.EnumerateArray()) + { + path.ValueKind.Should().Be(JsonValueKind.Array, $"{id} each path must be an array"); + path.GetArrayLength().Should().BeGreaterThan(0, $"{id} each path must contain at least one symbol"); + + foreach (var segment in path.EnumerateArray()) + { + segment.ValueKind.Should().Be(JsonValueKind.String, $"{id} path segments must be strings"); + segment.GetString().Should().NotBeNullOrWhiteSpace($"{id} path segments must be non-empty strings"); + } + } } } } diff --git a/tests/reachability/StellaOps.Reachability.FixtureTests/FixtureCoverageTests.cs b/tests/reachability/StellaOps.Reachability.FixtureTests/FixtureCoverageTests.cs new file mode 100644 index 000000000..01c2550cf --- /dev/null +++ b/tests/reachability/StellaOps.Reachability.FixtureTests/FixtureCoverageTests.cs @@ -0,0 +1,56 @@ +using System.Text.Json; +using FluentAssertions; +using Xunit; + +namespace StellaOps.Reachability.FixtureTests; + +public sealed class FixtureCoverageTests +{ + private static readonly string RepoRoot = ReachbenchFixtureTests.LocateRepoRoot(); + private static readonly string ReachabilityRoot = Path.Combine(RepoRoot, "tests", "reachability"); + private static readonly string CorpusRoot = Path.Combine(ReachabilityRoot, "corpus"); + private static readonly string SamplesPublicRoot = Path.Combine(ReachabilityRoot, "samples-public"); + + [Fact] + public void CorpusAndPublicSamplesCoverExpectedLanguageBuckets() + { + var corpusLanguages = ReadManifestLanguages(Path.Combine(CorpusRoot, "manifest.json")); + corpusLanguages.Should().Contain(new[] { "dotnet", "go", "python", "rust" }); + + var samplesLanguages = ReadManifestLanguages(Path.Combine(SamplesPublicRoot, "manifest.json")); + samplesLanguages.Should().Contain(new[] { "csharp", "js", "php" }); + } + + [Fact] + public void CorpusManifestIsSorted() + { + var keys = ReadManifestKeys(Path.Combine(CorpusRoot, "manifest.json")); + keys.Should().NotBeEmpty("corpus manifest should have entries"); + keys.Should().BeInAscendingOrder(StringComparer.Ordinal); + } + + private static string[] ReadManifestLanguages(string manifestPath) + { + File.Exists(manifestPath).Should().BeTrue($"{manifestPath} should exist"); + + using var doc = JsonDocument.Parse(File.ReadAllBytes(manifestPath)); + return doc.RootElement.EnumerateArray() + .Select(entry => entry.GetProperty("language").GetString()) + .Where(language => !string.IsNullOrWhiteSpace(language)) + .Select(language => language!) + .Distinct(StringComparer.Ordinal) + .OrderBy(language => language, StringComparer.Ordinal) + .ToArray(); + } + + private static string[] ReadManifestKeys(string manifestPath) + { + File.Exists(manifestPath).Should().BeTrue($"{manifestPath} should exist"); + + using var doc = JsonDocument.Parse(File.ReadAllBytes(manifestPath)); + return doc.RootElement.EnumerateArray() + .Select(entry => $"{entry.GetProperty("language").GetString()}/{entry.GetProperty("id").GetString()}") + .ToArray(); + } +} + diff --git a/tests/reachability/StellaOps.Reachability.FixtureTests/ReachabilityReplayWriterTests.cs b/tests/reachability/StellaOps.Reachability.FixtureTests/ReachabilityReplayWriterTests.cs index f2247c7ca..0b462cfaf 100644 --- a/tests/reachability/StellaOps.Reachability.FixtureTests/ReachabilityReplayWriterTests.cs +++ b/tests/reachability/StellaOps.Reachability.FixtureTests/ReachabilityReplayWriterTests.cs @@ -32,7 +32,7 @@ public sealed class ReachabilityReplayWriterTests new("zastava", "cas://trace/1", "FFEE", DateTimeOffset.Parse("2025-10-15T09:00:00Z", CultureInfo.InvariantCulture)) // duplicate once normalized }; - var writer = new ReachabilityReplayWriter(); + var writer = new StellaOps.Scanner.Reachability.ReachabilityReplayWriter(); writer.AttachEvidence(manifest, graphs, traces); manifest.Reachability.Should().NotBeNull(); @@ -52,10 +52,12 @@ public sealed class ReachabilityReplayWriterTests public void AttachEvidence_DoesNotCreateSectionWhenEmpty() { var manifest = new ReplayManifest(); - var writer = new ReachabilityReplayWriter(); + var writer = new StellaOps.Scanner.Reachability.ReachabilityReplayWriter(); writer.AttachEvidence(manifest, Array.Empty(), Array.Empty()); - manifest.Reachability.Should().BeNull(); + manifest.Reachability.AnalysisId.Should().BeNull(); + manifest.Reachability.Graphs.Should().BeEmpty(); + manifest.Reachability.RuntimeTraces.Should().BeEmpty(); } } diff --git a/tests/reachability/StellaOps.Replay.Core.Tests/ReplayMongoModelsTests.cs b/tests/reachability/StellaOps.Replay.Core.Tests/ReplayMongoModelsTests.cs deleted file mode 100644 index 87bf2d3c4..000000000 --- a/tests/reachability/StellaOps.Replay.Core.Tests/ReplayMongoModelsTests.cs +++ /dev/null @@ -1,57 +0,0 @@ -using FluentAssertions; -using MongoDB.Bson.Serialization; -using StellaOps.Replay.Core; -using Xunit; - -namespace StellaOps.Replay.Core.Tests; - -public sealed class ReplayMongoModelsTests -{ - [Fact] - public void ReplayRunRecord_SerializesWithExpectedFields() - { - var record = new ReplayRunRecord - { - Id = "scan-1", - ManifestHash = "sha256:abc", - Status = "verified", - Outputs = new ReplayRunOutputs { Sbom = "sha256:sbom", Findings = "sha256:findings", Vex = "sha256:vex" }, - Signatures = new() { new ReplaySignatureRecord { Profile = "FIPS", Verified = true } } - }; - - var bson = record.ToBsonDocument(); - - bson.Should().ContainKey("_id"); - bson["manifestHash"].AsString.Should().Be("sha256:abc"); - bson["status"].AsString.Should().Be("verified"); - bson["outputs"].AsBsonDocument["sbom"].AsString.Should().Be("sha256:sbom"); - bson["signatures"].AsBsonArray.Should().HaveCount(1); - } - - [Fact] - public void ReplayBundleRecord_UsesIdAsDigest() - { - var record = new ReplayBundleRecord { Id = "abc", Type = "input", Size = 10, Location = "cas://replay/ab/abc.tar.zst" }; - - var bson = record.ToBsonDocument(); - bson["_id"].AsString.Should().Be("abc"); - bson["type"].AsString.Should().Be("input"); - } - - [Fact] - public void ReplaySubjectRecord_StoresLayers() - { - var record = new ReplaySubjectRecord - { - OciDigest = "sha256:img", - Layers = new() - { - new ReplayLayerRecord { LayerDigest = "l1", MerkleRoot = "m1", LeafCount = 2 }, - new ReplayLayerRecord { LayerDigest = "l2", MerkleRoot = "m2", LeafCount = 3 } - } - }; - - var doc = record.ToBsonDocument(); - doc["layers"].AsBsonArray.Should().HaveCount(2); - } -} diff --git a/tests/reachability/StellaOps.ScannerSignals.IntegrationTests/ScannerToSignalsReachabilityTests.cs b/tests/reachability/StellaOps.ScannerSignals.IntegrationTests/ScannerToSignalsReachabilityTests.cs index a9ef8b2a1..31f838ada 100644 --- a/tests/reachability/StellaOps.ScannerSignals.IntegrationTests/ScannerToSignalsReachabilityTests.cs +++ b/tests/reachability/StellaOps.ScannerSignals.IntegrationTests/ScannerToSignalsReachabilityTests.cs @@ -10,7 +10,6 @@ using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using MongoDB.Bson; using StellaOps.Scanner.Reachability; using StellaOps.Signals.Models; using StellaOps.Signals.Options; @@ -36,7 +35,25 @@ public sealed class ScannerToSignalsReachabilityTests var variantPath = Path.Combine(FixtureRoot, caseId, "images", variant); Directory.Exists(variantPath).Should().BeTrue(); - var builder = ReachabilityGraphBuilder.FromFixture(variantPath); + var truth = JsonDocument.Parse(File.ReadAllText(Path.Combine(variantPath, "reachgraph.truth.json"))).RootElement; + var paths = truth.GetProperty("paths") + .EnumerateArray() + .Select(path => path.EnumerateArray().Select(x => x.GetString()!).Where(x => !string.IsNullOrWhiteSpace(x)).ToList()) + .Where(path => path.Count > 0) + .ToList(); + + var builder = new ReachabilityGraphBuilder(); + foreach (var path in paths) + { + for (var i = 0; i < path.Count; i++) + { + builder.AddNode(path[i]); + if (i + 1 < path.Count) + { + builder.AddEdge(path[i], path[i + 1]); + } + } + } var artifactJson = builder.BuildJson(indented: false); var parser = new SimpleJsonCallgraphParser("java"); var parserResolver = new StaticParserResolver(new Dictionary @@ -45,10 +62,12 @@ public sealed class ScannerToSignalsReachabilityTests }); var artifactStore = new InMemoryCallgraphArtifactStore(); var callgraphRepo = new InMemoryCallgraphRepository(); + var reachabilityStore = new InMemoryReachabilityStoreRepository(TimeProvider.System); var ingestionService = new CallgraphIngestionService( parserResolver, artifactStore, callgraphRepo, + reachabilityStore, new CallgraphNormalizationService(), Options.Create(new SignalsOptions()), TimeProvider.System, @@ -77,12 +96,14 @@ public sealed class ScannerToSignalsReachabilityTests new NullEventsPublisher(), NullLogger.Instance); - var truth = JsonDocument.Parse(File.ReadAllText(Path.Combine(variantPath, "reachgraph.truth.json"))).RootElement; - var entryPoints = truth.GetProperty("paths").EnumerateArray() - .Select(path => path[0].GetString()!) + var entryPoints = paths + .Select(path => path[0]) + .Distinct(StringComparer.Ordinal) + .ToList(); + var targets = paths + .Select(path => path[^1]) .Distinct(StringComparer.Ordinal) .ToList(); - var targets = truth.GetProperty("sinks").EnumerateArray().Select(s => s.GetProperty("sid").GetString()!).ToList(); var recomputeRequest = new ReachabilityRecomputeRequest { @@ -161,7 +182,7 @@ public sealed class ScannerToSignalsReachabilityTests { if (string.IsNullOrWhiteSpace(document.Id)) { - document.Id = ObjectId.GenerateNewId().ToString(); + document.Id = $"cg-{storage.Count + 1}"; } storage[document.Id] = document; @@ -228,6 +249,9 @@ public sealed class ScannerToSignalsReachabilityTests private sealed class InMemoryCallgraphArtifactStore : ICallgraphArtifactStore { + private readonly Dictionary artifacts = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary manifests = new(StringComparer.OrdinalIgnoreCase); + public async Task SaveAsync(CallgraphArtifactSaveRequest request, Stream content, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(request); @@ -251,6 +275,15 @@ public sealed class ScannerToSignalsReachabilityTests var casUri = $"cas://fixtures/{request.Component}/{request.Version}/{computedHash}"; var manifestPath = $"cas://fixtures/{request.Component}/{request.Version}/{computedHash}/manifest"; + artifacts[computedHash] = bytes; + + if (request.ManifestContent is not null) + { + await using var manifestBuffer = new MemoryStream(); + await request.ManifestContent.CopyToAsync(manifestBuffer, cancellationToken).ConfigureAwait(false); + manifests[computedHash] = manifestBuffer.ToArray(); + } + return new StoredCallgraphArtifact( Path: $"fixtures/{request.Component}/{request.Version}/{request.FileName}", Length: bytes.Length, @@ -260,6 +293,29 @@ public sealed class ScannerToSignalsReachabilityTests ManifestPath: manifestPath, ManifestCasUri: manifestPath); } + + public Task GetAsync(string hash, string? fileName, CancellationToken cancellationToken) + { + if (!artifacts.TryGetValue(hash, out var bytes)) + { + return Task.FromResult(null); + } + + return Task.FromResult(new MemoryStream(bytes, writable: false)); + } + + public Task GetManifestAsync(string hash, CancellationToken cancellationToken) + { + if (!manifests.TryGetValue(hash, out var bytes)) + { + return Task.FromResult(null); + } + + return Task.FromResult(new MemoryStream(bytes, writable: false)); + } + + public Task ExistsAsync(string hash, CancellationToken cancellationToken) + => Task.FromResult(artifacts.ContainsKey(hash)); } private static string LocateRepoRoot() { diff --git a/tests/reachability/StellaOps.Signals.Reachability.Tests/ReachabilityScoringTests.cs b/tests/reachability/StellaOps.Signals.Reachability.Tests/ReachabilityScoringTests.cs index 69c268ec8..a9b8e5043 100644 --- a/tests/reachability/StellaOps.Signals.Reachability.Tests/ReachabilityScoringTests.cs +++ b/tests/reachability/StellaOps.Signals.Reachability.Tests/ReachabilityScoringTests.cs @@ -8,7 +8,6 @@ using System.Threading.Tasks; using FluentAssertions; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using MongoDB.Bson; using StellaOps.Signals.Models; using StellaOps.Signals.Options; using StellaOps.Signals.Parsing; @@ -47,15 +46,31 @@ public sealed class ReachabilityScoringTests public async Task RecomputedFactsMatchTruthFixtures(string caseId, string variant) { var casePath = Path.Combine(FixtureRoot, caseId); - var variantPath = Path.Combine(casePath, "images", variant); - var truth = JsonDocument.Parse(File.ReadAllText(Path.Combine(variantPath, "reachgraph.truth.json"))).RootElement; - var sinks = truth.GetProperty("sinks").EnumerateArray().Select(x => x.GetProperty("sid").GetString()!).ToList(); - var entryPoints = truth.GetProperty("paths").EnumerateArray() - .Select(path => path[0].GetString()!) + var caseJson = JsonDocument.Parse(File.ReadAllText(Path.Combine(casePath, "case.json"))).RootElement; + var reachablePathsNode = caseJson + .GetProperty("ground_truth") + .GetProperty("reachable_variant") + .GetProperty("evidence") + .GetProperty("paths"); + + var paths = reachablePathsNode.EnumerateArray() + .Select(path => path.EnumerateArray().Select(x => x.GetString()!).Where(x => !string.IsNullOrWhiteSpace(x)).ToList()) + .Where(path => path.Count > 0) + .ToList(); + + var entryPoints = paths + .Select(path => path[0]) + .Where(p => !string.IsNullOrWhiteSpace(p)) .Distinct(StringComparer.Ordinal) .ToList(); - var callgraph = await LoadCallgraphAsync(caseId, variant, variantPath); + var sinks = paths + .Select(path => path[^1]) + .Where(p => !string.IsNullOrWhiteSpace(p)) + .Distinct(StringComparer.Ordinal) + .ToList(); + + var callgraph = BuildCallgraphFromPaths(caseId, paths); var callgraphRepo = new InMemoryCallgraphRepository(callgraph); var factRepo = new InMemoryReachabilityFactRepository(); var options = new SignalsOptions(); @@ -66,7 +81,7 @@ public sealed class ReachabilityScoringTests callgraphRepo, factRepo, TimeProvider.System, - Options.Create(options), + Microsoft.Extensions.Options.Options.Create(options), cache, unknowns, eventsPublisher, @@ -149,41 +164,46 @@ public sealed class ReachabilityScoringTests }; } - private static async Task LoadCallgraphAsync(string caseId, string variant, string variantPath) + private static CallgraphDocument BuildCallgraphFromPaths(string caseId, IReadOnlyList> paths) { - var parser = new SimpleJsonCallgraphParser("fixture"); var nodes = new Dictionary(StringComparer.Ordinal); var edges = new List(); - foreach (var fileName in new[] { "callgraph.static.json", "callgraph.framework.json" }) + foreach (var path in paths) { - var path = Path.Combine(variantPath, fileName); - if (!File.Exists(path)) + if (path.Count == 0) { continue; } - await using var stream = File.OpenRead(path); - var result = await parser.ParseAsync(stream, CancellationToken.None); - foreach (var node in result.Nodes) + foreach (var nodeId in path) { - nodes[node.Id] = node; + if (!nodes.ContainsKey(nodeId)) + { + nodes[nodeId] = new CallgraphNode(nodeId, nodeId, "function", null, null, null); + } } - edges.AddRange(result.Edges); + for (var i = 0; i < path.Count - 1; i++) + { + edges.Add(new CallgraphEdge(path[i], path[i + 1], "call")); + } } return new CallgraphDocument { - Id = ObjectId.GenerateNewId().ToString(), + Id = caseId, Language = "fixture", Component = caseId, - Version = variant, - Nodes = nodes.Values.ToList(), - Edges = edges, + Version = "truth", + Nodes = nodes.Values.OrderBy(n => n.Id, StringComparer.Ordinal).ToList(), + Edges = edges + .OrderBy(e => e.SourceId, StringComparer.Ordinal) + .ThenBy(e => e.TargetId, StringComparer.Ordinal) + .ToList(), Artifact = new CallgraphArtifactMetadata { - Path = $"cas://fixtures/{caseId}/{variant}", + Path = $"cas://fixtures/{caseId}", Hash = "stub", ContentType = "application/json", Length = 0 diff --git a/tests/reachability/StellaOps.Signals.Reachability.Tests/RuntimeFactsIngestionServiceTests.cs b/tests/reachability/StellaOps.Signals.Reachability.Tests/RuntimeFactsIngestionServiceTests.cs index de739a1ea..ce28f4f58 100644 --- a/tests/reachability/StellaOps.Signals.Reachability.Tests/RuntimeFactsIngestionServiceTests.cs +++ b/tests/reachability/StellaOps.Signals.Reachability.Tests/RuntimeFactsIngestionServiceTests.cs @@ -15,12 +15,23 @@ namespace StellaOps.Signals.Reachability.Tests; public sealed class RuntimeFactsIngestionServiceTests { private readonly FakeReachabilityFactRepository repository = new(); + private readonly FakeReachabilityCache cache = new(); + private readonly FakeEventsPublisher eventsPublisher = new(); + private readonly FakeScoringService scoringService = new(); + private readonly FakeProvenanceNormalizer provenanceNormalizer = new(); private readonly FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2025-11-09T10:15:00Z", null, System.Globalization.DateTimeStyles.AssumeUniversal)); private readonly RuntimeFactsIngestionService sut; public RuntimeFactsIngestionServiceTests() { - sut = new RuntimeFactsIngestionService(repository, timeProvider, NullLogger.Instance); + sut = new RuntimeFactsIngestionService( + repository, + timeProvider, + cache, + eventsPublisher, + scoringService, + provenanceNormalizer, + NullLogger.Instance); } [Fact] @@ -145,4 +156,83 @@ public sealed class RuntimeFactsIngestionServiceTests public Task GetBySubjectAsync(string subjectKey, CancellationToken cancellationToken) => Task.FromResult(LastUpsert is { SubjectKey: not null } doc && doc.SubjectKey == subjectKey ? doc : null); } + + private sealed class FakeReachabilityCache : IReachabilityCache + { + private readonly Dictionary storage = new(StringComparer.Ordinal); + + public Task GetAsync(string subjectKey, CancellationToken cancellationToken) + { + storage.TryGetValue(subjectKey, out var document); + return Task.FromResult(document); + } + + public Task SetAsync(ReachabilityFactDocument document, CancellationToken cancellationToken) + { + storage[document.SubjectKey] = document; + return Task.CompletedTask; + } + + public Task InvalidateAsync(string subjectKey, CancellationToken cancellationToken) + { + storage.Remove(subjectKey); + return Task.CompletedTask; + } + } + + private sealed class FakeEventsPublisher : IEventsPublisher + { + public List Published { get; } = new(); + + public Task PublishFactUpdatedAsync(ReachabilityFactDocument fact, CancellationToken cancellationToken) + { + Published.Add(fact); + return Task.CompletedTask; + } + } + + private sealed class FakeScoringService : IReachabilityScoringService + { + public List Requests { get; } = new(); + + public Task RecomputeAsync(ReachabilityRecomputeRequest request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(new ReachabilityFactDocument + { + Subject = request.Subject, + SubjectKey = request.Subject.ToSubjectKey(), + CallgraphId = request.CallgraphId, + ComputedAt = TimeProvider.System.GetUtcNow() + }); + } + } + + private sealed class FakeProvenanceNormalizer : IRuntimeFactsProvenanceNormalizer + { + public ProvenanceFeed NormalizeToFeed( + IEnumerable events, + ReachabilitySubject subject, + string callgraphId, + Dictionary? metadata, + DateTimeOffset generatedAt) => new() + { + FeedId = "fixture", + GeneratedAt = generatedAt, + CorrelationId = callgraphId, + Records = new List() + }; + + public ContextFacts CreateContextFacts( + IEnumerable events, + ReachabilitySubject subject, + string callgraphId, + Dictionary? metadata, + DateTimeOffset timestamp) => new() + { + Provenance = NormalizeToFeed(events, subject, callgraphId, metadata, timestamp), + LastUpdatedAt = timestamp, + RecordCount = events is ICollection collection ? collection.Count : 0 + }; + } } diff --git a/tests/reachability/corpus/README.md b/tests/reachability/corpus/README.md index 8bc7897a7..4f4548669 100644 --- a/tests/reachability/corpus/README.md +++ b/tests/reachability/corpus/README.md @@ -2,9 +2,10 @@ Layout - `manifest.json` — deterministic SHA-256 hashes for each case file. -- `//expect.yaml` — state (`reachable|conditional|unreachable`), score, evidence refs. +- `//ground-truth.json` — expected reachability outcome (`reachable|unreachable`) and example path(s) (Reachbench truth schema v1). - `//callgraph.static.json` — static call graph sample (stub for MVP). - `//vex.openvex.json` — expected VEX slice for the case. +- Legacy `expect.yaml` has been retired; its state/score are preserved under `legacy_expect` in `ground-truth.json`. Determinism - JSON files have sorted keys; hashes recorded in `manifest.json`. diff --git a/tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/expect.yaml b/tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/expect.yaml deleted file mode 100644 index a4c716466..000000000 --- a/tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/expect.yaml +++ /dev/null @@ -1,11 +0,0 @@ -schema_version: reach-corpus.expect/v1 -id: dotnet-kestrel-CVE-2023-44487-http2-rapid-reset -language: dotnet -state: reachable -score: 0.85 -static_evidence: - callgraphs: - - callgraph.static.json -runtime_evidence: [] -vex: vex.openvex.json -notes: "MVP fixture stub; replace with real callgraph and traces when available." diff --git a/tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/ground-truth.json b/tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/ground-truth.json new file mode 100644 index 000000000..4ed92b766 --- /dev/null +++ b/tests/reachability/corpus/dotnet/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/ground-truth.json @@ -0,0 +1,16 @@ +{ + "case_id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset", + "legacy_expect": { + "schema_version": "reach-corpus.expect/v1", + "score": 0.85, + "state": "reachable" + }, + "paths": [ + [ + "sym://dotnet:entry", + "sym://dotnet:sink" + ] + ], + "schema_version": "reachbench.reachgraph.truth/v1", + "variant": "reachable" +} diff --git a/tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/expect.yaml b/tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/expect.yaml deleted file mode 100644 index bab5005c6..000000000 --- a/tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/expect.yaml +++ /dev/null @@ -1,11 +0,0 @@ -schema_version: reach-corpus.expect/v1 -id: go-ssh-CVE-2020-9283-keyexchange -language: go -state: reachable -score: 0.80 -static_evidence: - callgraphs: - - callgraph.static.json -runtime_evidence: [] -vex: vex.openvex.json -notes: "MVP fixture stub; replace with real callgraph and traces when available." diff --git a/tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/ground-truth.json b/tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/ground-truth.json new file mode 100644 index 000000000..6f8be5515 --- /dev/null +++ b/tests/reachability/corpus/go/go-ssh-CVE-2020-9283-keyexchange/ground-truth.json @@ -0,0 +1,16 @@ +{ + "case_id": "go-ssh-CVE-2020-9283-keyexchange", + "legacy_expect": { + "schema_version": "reach-corpus.expect/v1", + "score": 0.8, + "state": "reachable" + }, + "paths": [ + [ + "sym://go:entry", + "sym://go:sink" + ] + ], + "schema_version": "reachbench.reachgraph.truth/v1", + "variant": "reachable" +} diff --git a/tests/reachability/corpus/manifest.json b/tests/reachability/corpus/manifest.json index 65f5f32ba..fa2c4768b 100644 --- a/tests/reachability/corpus/manifest.json +++ b/tests/reachability/corpus/manifest.json @@ -1,38 +1,38 @@ [ { "files": { - "callgraph.static.json": "b2f32c667c8ec76d50d2b106dc055777f0135e2cf6938540fd1840eda82b4fe7", - "expect.yaml": "ffbbd4d12e2ee1898db9c34556754df8b7e1b21208298831714ee5e18ff4637d", - "vex.openvex.json": "e8cb5215049b9b1fe76354da6f67e8a5ef336a49780a0881e50b85d3ac526e63" + "callgraph.static.json": "7359d8c26f16151a4b05cf0e6675e5c66b5ffb6396b906e74c0d5bb2f290e972", + "ground-truth.json": "5e9fe73eabe607c9912c64d7b3d31b456a2b74631b935ce81f769d4520303c59", + "vex.openvex.json": "c3593790f769974b1b66aa5331f1d3ad4d699f77f198b2e77e78659ee79d3c15" }, "id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset", "language": "dotnet" }, { "files": { - "callgraph.static.json": "b2f32c667c8ec76d50d2b106dc055777f0135e2cf6938540fd1840eda82b4fe7", - "expect.yaml": "e9b38f76f0814b90c401368335cc953afc511d2256f3bfa76a84928175b506ac", - "vex.openvex.json": "e8cb5215049b9b1fe76354da6f67e8a5ef336a49780a0881e50b85d3ac526e63" + "callgraph.static.json": "7359d8c26f16151a4b05cf0e6675e5c66b5ffb6396b906e74c0d5bb2f290e972", + "ground-truth.json": "430adb2d001b526cff666336689006bad00e27c9f82582795a2d9dd106e1797d", + "vex.openvex.json": "c3593790f769974b1b66aa5331f1d3ad4d699f77f198b2e77e78659ee79d3c15" }, "id": "go-ssh-CVE-2020-9283-keyexchange", "language": "go" }, { "files": { - "callgraph.static.json": "b2f32c667c8ec76d50d2b106dc055777f0135e2cf6938540fd1840eda82b4fe7", - "expect.yaml": "381e9379618014f346e11462ffe79b22785113f05def078bf85c26fe7a696830", - "vex.openvex.json": "e8cb5215049b9b1fe76354da6f67e8a5ef336a49780a0881e50b85d3ac526e63" + "callgraph.static.json": "7359d8c26f16151a4b05cf0e6675e5c66b5ffb6396b906e74c0d5bb2f290e972", + "ground-truth.json": "50538def2e0a8b28134051b52a848eb4b53d43cf7a6eb6d041e8fc9f1d9210f1", + "vex.openvex.json": "c3593790f769974b1b66aa5331f1d3ad4d699f77f198b2e77e78659ee79d3c15" }, "id": "python-django-CVE-2019-19844-sqli-like", "language": "python" }, { "files": { - "callgraph.static.json": "b2f32c667c8ec76d50d2b106dc055777f0135e2cf6938540fd1840eda82b4fe7", - "expect.yaml": "6c3bd42fd80277b874021b2f1a43133a9365ad298428b202d75970037de5d95f", - "vex.openvex.json": "e8cb5215049b9b1fe76354da6f67e8a5ef336a49780a0881e50b85d3ac526e63" + "callgraph.static.json": "7359d8c26f16151a4b05cf0e6675e5c66b5ffb6396b906e74c0d5bb2f290e972", + "ground-truth.json": "36312fc03b7f46c8655c21448c9fb7acd6495344896b79010fbd9644a182a865", + "vex.openvex.json": "c3593790f769974b1b66aa5331f1d3ad4d699f77f198b2e77e78659ee79d3c15" }, "id": "rust-axum-header-parsing-TBD", "language": "rust" } -] \ No newline at end of file +] diff --git a/tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/expect.yaml b/tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/expect.yaml deleted file mode 100644 index 75c10b7ee..000000000 --- a/tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/expect.yaml +++ /dev/null @@ -1,11 +0,0 @@ -schema_version: reach-corpus.expect/v1 -id: python-django-CVE-2019-19844-sqli-like -language: python -state: reachable -score: 0.80 -static_evidence: - callgraphs: - - callgraph.static.json -runtime_evidence: [] -vex: vex.openvex.json -notes: "MVP fixture stub; replace with real callgraph and traces when available." diff --git a/tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/ground-truth.json b/tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/ground-truth.json new file mode 100644 index 000000000..395709c56 --- /dev/null +++ b/tests/reachability/corpus/python/python-django-CVE-2019-19844-sqli-like/ground-truth.json @@ -0,0 +1,16 @@ +{ + "case_id": "python-django-CVE-2019-19844-sqli-like", + "legacy_expect": { + "schema_version": "reach-corpus.expect/v1", + "score": 0.8, + "state": "reachable" + }, + "paths": [ + [ + "sym://python:entry", + "sym://python:sink" + ] + ], + "schema_version": "reachbench.reachgraph.truth/v1", + "variant": "reachable" +} diff --git a/tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/expect.yaml b/tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/expect.yaml deleted file mode 100644 index d337ec293..000000000 --- a/tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/expect.yaml +++ /dev/null @@ -1,11 +0,0 @@ -schema_version: reach-corpus.expect/v1 -id: rust-axum-header-parsing-TBD -language: rust -state: conditional -score: 0.60 -static_evidence: - callgraphs: - - callgraph.static.json -runtime_evidence: [] -vex: vex.openvex.json -notes: "MVP fixture stub; replace with real callgraph and traces when available." diff --git a/tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/ground-truth.json b/tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/ground-truth.json new file mode 100644 index 000000000..b8ca6f242 --- /dev/null +++ b/tests/reachability/corpus/rust/rust-axum-header-parsing-TBD/ground-truth.json @@ -0,0 +1,11 @@ +{ + "case_id": "rust-axum-header-parsing-TBD", + "legacy_expect": { + "schema_version": "reach-corpus.expect/v1", + "score": 0.6, + "state": "conditional" + }, + "paths": [], + "schema_version": "reachbench.reachgraph.truth/v1", + "variant": "unreachable" +} diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/reachable/manifest.json index ab95ecdf9..27364556d 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "curl-CVE-2023-38545-socks5-heap", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "73617ac40fc52b1f17fb157cc9b3b3112af81efc299a551fab811dae272ff905", - "sbom.cdx.json": "c89dcfe1faad15e6cd441bc5d4a0269b9586238750c5b93350660cd0603e3318", - "sbom.spdx.json": "abc7d79dd5ef2df5b2a3287fa761912cf04e7f5107b268fc96dfff157b92fdca", - "symbols.json": "a701cf77e8bf77b8b46618bb9ed16938aa7c5fdefdcd56a4a21722622a711470", - "vex.openvex.json": "e697ff8a01a9217f97736306fdfe11a323d1fda3c79e41e10fd7c18cbc4ba601" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "9545261d413f4f85d120ebe8432c32ba97ba3feb2d34075fd689fcb5794f3ab0", + "sbom.cdx.json": "ce41fd9b9edadf94a8cc84a3cce4e175b0602fd2e0d8dcb067273b9584479980", + "sbom.spdx.json": "10d7417961d3cac0f3a5c4b083917fba3dc4f9bd9140d80aad0a873435158482", + "symbols.json": "c5f473aff5b428df5a3f9c3393b7fbceb94214e3c2fd4f547d4f258ca25a3080", + "vex.openvex.json": "0518d09c2ae692b96553feb821ff8138fc0ea6c840d75c1f80149add21127ddd" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/unreachable/manifest.json index 09dd24195..14bcbcaa4 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/curl-CVE-2023-38545-socks5-heap/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "curl-CVE-2023-38545-socks5-heap", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "835f69f6cec3472d786be6cd1b66062dfef84dea87e1885023a10f77a9cf5c85", - "sbom.cdx.json": "c89dcfe1faad15e6cd441bc5d4a0269b9586238750c5b93350660cd0603e3318", - "sbom.spdx.json": "abc7d79dd5ef2df5b2a3287fa761912cf04e7f5107b268fc96dfff157b92fdca", - "symbols.json": "a9c96e72421bfc775df8cc7cf7203eda01e7ce9101ec6eff03decc5960fe3c1a", - "vex.openvex.json": "6ce74c0c7b7b9502a189334aea3e0c4b03c7d396bff553697dc41ae3d8eb21de" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "490c4175eb06e0c623e60263d2ce029ffa8b236aea5780c448b8180f38a1bf6f", + "sbom.cdx.json": "ce41fd9b9edadf94a8cc84a3cce4e175b0602fd2e0d8dcb067273b9584479980", + "sbom.spdx.json": "10d7417961d3cac0f3a5c4b083917fba3dc4f9bd9140d80aad0a873435158482", + "symbols.json": "1b6a9e5598d2521e0ca55ed0f3f287ef19dc11cb1fb24fe961370c2fa7036214", + "vex.openvex.json": "a9fa7e917601538e17750fb1c25b24e18333c779ec0d5d98d4fbccf84e2f544e" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/reachable/manifest.json index 0a01b95ad..938908204 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "ea22d69670f767c677395af15b2b6098764b9ae3f5dd94c229886c22011a8a98", - "sbom.cdx.json": "4728c4bbdafe8f157820bb8559c13105aba373526699695b5ee53283ef761582", - "sbom.spdx.json": "84d8e2bd7b0fcc802cc91f21999908f1ae55659492969d48944a9652cbad0e7c", - "symbols.json": "77c5d912fd82799e53f3082fd3cb21074ef9962e1809ef6f045da84dadc9516d", - "vex.openvex.json": "d5c30749ed5dad4f2337f482d591e9755eb95efec99fdc9727fc8b7af1d13337" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "5396e1c97612e0963bdaf9d5d3f570f095feaccfd46ed6e96af52a6dc4608608", + "sbom.cdx.json": "8747790b2c9638b08aedca818367852889ee9bb50f1be1212b9c46b27296b8b9", + "sbom.spdx.json": "fd5b8befa1a59f06c315406213426ee516276ad806f4acb1f53472149d97c402", + "symbols.json": "c2bc2c131db1565b272900b2d86733086d601fc05a9072a43b9cd8b89a2e6f95", + "vex.openvex.json": "2bc0466a7b733a0915b6a799e91ec731c0700d5bea8645c0bf983b6da180bc48" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/unreachable/manifest.json index 6cf2cfcde..feac74e25 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-kestrel-CVE-2023-44487-http2-rapid-reset/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "dotnet-kestrel-CVE-2023-44487-http2-rapid-reset", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "049dbb44a4e56e34d28fedaf09e41a025a0427935ee3b60017c23183ee58cb29", - "sbom.cdx.json": "4728c4bbdafe8f157820bb8559c13105aba373526699695b5ee53283ef761582", - "sbom.spdx.json": "84d8e2bd7b0fcc802cc91f21999908f1ae55659492969d48944a9652cbad0e7c", - "symbols.json": "c983b416c449d0ff5735bb5cc7f1792e113ae465d2f76aa3512a58d0d44b6962", - "vex.openvex.json": "5168a01bbbfe00f76ab536d818b9e2f156d37ffc01ffd278a52bebacbd6ee295" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "86a0dad5b06b69018a35931b1ef8fb700abe6511f75aa81dcffc23f0411cc086", + "sbom.cdx.json": "8747790b2c9638b08aedca818367852889ee9bb50f1be1212b9c46b27296b8b9", + "sbom.spdx.json": "fd5b8befa1a59f06c315406213426ee516276ad806f4acb1f53472149d97c402", + "symbols.json": "0793a11190a789d63cac1d15ae259dcbe48764dd0f75000176e3abf8f3a3beb6", + "vex.openvex.json": "cd54fe28bf7f171a2a47e6118b05ad26013a32d97e2b9eef143eab75208d9fa4" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/reachable/manifest.json index 2f3182519..691e0d727 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "dotnet-newtonsoft-deser-TBD", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "0cb6f48faa72b41620612610e5e2e4f2ec03ec1a4e843a2e50437e9a95d5d261", - "sbom.cdx.json": "f7a517a7608f0216f1596811a0184c6df1ccf6e71f88aff26e4ddf0e0ca1d9ff", - "sbom.spdx.json": "c24a0ec0df18f1dea45f0afcb87fac8b17d1d0d87d711c9e9610244dc2ac9747", - "symbols.json": "bcbace5bc3c071f4a751844420a948b3932657dc287191957ea5247349e68e32", - "vex.openvex.json": "dafc01265689557670f9dc30874186942a018e9c095f64a03bb4b8d53e1e08d3" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "7c1b7d56df4efc97360ba7754feb1051644e624afa2589971fab09507827e677", + "sbom.cdx.json": "c7283a731ca81300f6cda9e944451062a92c7eb0559ebdc6b96f6afeea637187", + "sbom.spdx.json": "da4978369cae300336e4abd570edb8c8de27bcb5ff2c5131975cae7d8ee01f8e", + "symbols.json": "d03361b683ae570864824a8e57c91ca875590373d949d2f706af488c4ccbcc01", + "vex.openvex.json": "41e52bf3c0b40ca614d32f5c9b719b68c53e2a0f08f483d6c429120060c9d930" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/unreachable/manifest.json index 65dd901e0..cc40d74fa 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/dotnet-newtonsoft-deser-TBD/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "dotnet-newtonsoft-deser-TBD", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "1cdc387fc4487201eddb3d00e8d75deb053f87dff23a61cfe232d168e60ff91d", - "sbom.cdx.json": "f7a517a7608f0216f1596811a0184c6df1ccf6e71f88aff26e4ddf0e0ca1d9ff", - "sbom.spdx.json": "c24a0ec0df18f1dea45f0afcb87fac8b17d1d0d87d711c9e9610244dc2ac9747", - "symbols.json": "521cc59d537c4008afa37a1b8b379ede655f6619ee143bbae3123869fe12c653", - "vex.openvex.json": "a75b2a1ed3086773162b80e9fa307c2ab3dec809643c9bbd614432891b2bf5c2" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "aa1c4c8133ae26349e1a740293e875d91f3a5ba1b241eb39617a09ea1b6ced8e", + "sbom.cdx.json": "c7283a731ca81300f6cda9e944451062a92c7eb0559ebdc6b96f6afeea637187", + "sbom.spdx.json": "da4978369cae300336e4abd570edb8c8de27bcb5ff2c5131975cae7d8ee01f8e", + "symbols.json": "a804343735751e99bda81ce614d890fe19cb510bcb3d3b17dff05ab01decf2e1", + "vex.openvex.json": "65cdb8a5d02277eacf194c23cdb7a8adada7318f45f5ce4eb0e09fbcd9d8b615" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/reachable/manifest.json index c9d056206..227d089e2 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "glibc-CVE-2023-4911-looney-tunables", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "d5dea5b832043e3268aecda2a423c959718e645589535c04f71652dfa06f54d1", - "sbom.cdx.json": "e969033f28fe3debd8c93226cb7212ac2b0b54c8fc3e8a4d126d5dfb7bd8da1d", - "sbom.spdx.json": "08a961ada88b9d5706f5baba763ccfddf2a474b9faf0981f0a87a2933f7031df", - "symbols.json": "f9103b6f3df6caed469a68ec9677a34573e662de4ae61b7b1df630cbd2aab769", - "vex.openvex.json": "207691b26721a773abe2b9242afe5e7f68ace37c307262d09adc188df9c20dbd" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "f7200c066db6fefd2ed3168497ae7d8cb585f1d12479086217007df1bb2c1460", + "sbom.cdx.json": "e3bbce1051a27f877fdd76634902c835ac21a7f53241308878a404dbced491fc", + "sbom.spdx.json": "2b30ff6eabf0b4c5e76f2e5de6af21a6b48a746c51298a708a3674976ef5b8f8", + "symbols.json": "27dd785d49ef6b4229a0e5a25107346eea5cc8b7dd01c2fb9ba73b53456bcaee", + "vex.openvex.json": "bd6f67166fb31fa2a5e7211b71e083c8611f9c2b7d7e0607c31ce6df777a1f69" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/unreachable/manifest.json index f10bf5a03..b1e6f9fae 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/glibc-CVE-2023-4911-looney-tunables/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "glibc-CVE-2023-4911-looney-tunables", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "04554e2014584d380d3d113ab6a81de20ba9a2097e6b828849e4f8d748664b30", - "sbom.cdx.json": "e969033f28fe3debd8c93226cb7212ac2b0b54c8fc3e8a4d126d5dfb7bd8da1d", - "sbom.spdx.json": "08a961ada88b9d5706f5baba763ccfddf2a474b9faf0981f0a87a2933f7031df", - "symbols.json": "c328ec1c9620d072176a4409b740a6cf9dcba732e88cfa8427e40c17a76a498d", - "vex.openvex.json": "3131068dec558feffad9cd829c02dab9dcf8dd9ea19505ef1f2ee104ce89f13a" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "836f543e3e7b593582e2ffb529456ffc4309ec79d41e5f8b9eb5696f54d17883", + "sbom.cdx.json": "e3bbce1051a27f877fdd76634902c835ac21a7f53241308878a404dbced491fc", + "sbom.spdx.json": "2b30ff6eabf0b4c5e76f2e5de6af21a6b48a746c51298a708a3674976ef5b8f8", + "symbols.json": "fe742caccb2134c46594f3816b58b06f1cad6f2d62ea8dd55ad31ce4ce672906", + "vex.openvex.json": "3ebcafe7d9e0f211f80783568cd9bc4a92ddaa3609b2b0ef11471031246cadde" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/reachable/manifest.json index 213f395b5..17691e614 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "go-gateway-reflection-auth-bypass", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "4af3b748615b65895a3137f5781e8b8043afd2d28746b93ee5e4ab752efc4dd7", - "sbom.cdx.json": "033d6f89201c248b7d696af04d2dcf78434dd9f6ad8b64d5834d2fabe7ad5147", - "sbom.spdx.json": "3adb2a7c66b8135e3e96c113ee670a6932d24ec4737f52fd6156d3293de4f391", - "symbols.json": "55ce7298e1db590f299e9229b514a57b3ac6f461a876b07b1d80760c1df9cd5f", - "vex.openvex.json": "8252cba56e4b2a7785f0d62c6d3a2752f6ef27611fcfbb5808a3caac94c75097" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "f7c362965a307a6cf40f7921d2ad508cd503fa924ed3a391dba3afe54ab0dcdd", + "sbom.cdx.json": "16a041571c0641abe57929624e49f07353edb8980ecdd16340ef83f24f127cba", + "sbom.spdx.json": "8abd620f40a28d379b861d6ef640017ea119a8870890009dbd8126ed621a5c73", + "symbols.json": "dbf69a19ce1676cc809597ed9fce78c9fe8ebcf25186949a107971116a79a39b", + "vex.openvex.json": "b550e30451d7ef7ff612606711ecede1089d914bd8a26f5fbcf01ff1d4e36149" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/unreachable/manifest.json index 4d8ef8c05..4ba8bcd3f 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-gateway-reflection-auth-bypass/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "go-gateway-reflection-auth-bypass", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "b384a5d1977eab4f390c7112cc8006940d42701f7bfa71635c3e2edabc5929fa", - "sbom.cdx.json": "033d6f89201c248b7d696af04d2dcf78434dd9f6ad8b64d5834d2fabe7ad5147", - "sbom.spdx.json": "3adb2a7c66b8135e3e96c113ee670a6932d24ec4737f52fd6156d3293de4f391", - "symbols.json": "6ef5a5a9514afba7c768af1e27f9963b066aa7c4dc0295a436d8716e089a86dd", - "vex.openvex.json": "1a44ff9f0fa8098c5d74c12836d90efbe39dbf15a0360cf654b69e14fbf59cee" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "df9749530b5dc16127ab6782877e19e2bde09a40f7cd44edc8af327619498d32", + "sbom.cdx.json": "16a041571c0641abe57929624e49f07353edb8980ecdd16340ef83f24f127cba", + "sbom.spdx.json": "8abd620f40a28d379b861d6ef640017ea119a8870890009dbd8126ed621a5c73", + "symbols.json": "6571c9c658f4b0a967542a02cd5e5f4b82dd1ffaf7758c51d3ac9c2a83c6c86e", + "vex.openvex.json": "69ffc3f74db3d723a0354c0aa05f4e5920fdb02fc8ac72e9d82392b5997f074d" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/reachable/manifest.json index 50c10430e..31d29c117 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "go-ssh-CVE-2020-9283-keyexchange", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "d5e2ec246dbd03eb9af759e416bba5a00eb18a3dc344590b31e87bf69098d877", - "sbom.cdx.json": "1f685f8db5be777fe56e0d2f13769e2b7667bf7837b745e5c461175c1e3e6e1a", - "sbom.spdx.json": "ea729e7e6842f10375261f5f2fe949521a513adee0eb72a6f38ca5d2e93798a7", - "symbols.json": "4284612efe2a3770f720ffa490c5304a469baa36b5b7533f7ec9be24cf88be16", - "vex.openvex.json": "9a62366bb238e0ad0e025fdd675226e3959ffc1307f890d009a69212897d4b04" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "43fee4eeb52cec12879355873638959460eb91c463e2b2d3a67ef033f906469f", + "sbom.cdx.json": "a975829c9537c16db4d19306ba6bc809930b6ad9f96495a8202d59d3f174cf2c", + "sbom.spdx.json": "399d1f0946dfbe0fb66749f2b08df539f93285affbd059e0b66df55f485ed39a", + "symbols.json": "189002d4626708cdad2ff1bda786c47dd90002915f411324ad5dccbce65ba26d", + "vex.openvex.json": "1fdce721814a1a0c502882ab514ac7a361fdd3ea866869f4cf2c07578feb23d7" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/unreachable/manifest.json index 9628cb322..600bb1140 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/go-ssh-CVE-2020-9283-keyexchange/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "go-ssh-CVE-2020-9283-keyexchange", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "0fc5055224899bfc0fe645929dfc519e36fbdb2c939498fd22efc951f0b9f55c", - "sbom.cdx.json": "1f685f8db5be777fe56e0d2f13769e2b7667bf7837b745e5c461175c1e3e6e1a", - "sbom.spdx.json": "ea729e7e6842f10375261f5f2fe949521a513adee0eb72a6f38ca5d2e93798a7", - "symbols.json": "2f50e84f8b75e56d77574781a392743a564a6602d50b19ceb551f99051a42a4b", - "vex.openvex.json": "96f4f675a58cdf81002166b3d74b4a1481d9039593ed40b75e47e2c8ac262b5f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "ee1409484f2314be8471ebb0b1d3ab62d5bacbfd18dfc7380d9f94e2f214a6d4", + "sbom.cdx.json": "a975829c9537c16db4d19306ba6bc809930b6ad9f96495a8202d59d3f174cf2c", + "sbom.spdx.json": "399d1f0946dfbe0fb66749f2b08df539f93285affbd059e0b66df55f485ed39a", + "symbols.json": "b40d34be3d26d3293e9f06c21c58d1f89ef75897697207f71aa6e461cf9f72bf", + "vex.openvex.json": "537af070b5eb69fa842511fa63018ed6b8745631a156dcfc7abd1f60cc13e972" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/reachable/manifest.json index 842c04395..b5b81233a 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "java-jackson-CVE-2019-12384-polymorphic-deser", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "1a8a4447a07938fb604399b29f52c52fe8abcb5777fa94c32697be3b4936deb5", - "sbom.cdx.json": "bc802fbfc6cd9cbe431416484872cbbaafe2e1e4bb8d00d1437a8d48bfc2d2cc", - "sbom.spdx.json": "ffb869fd2e4d6d8bbad352d91a431f1038359995701dda2c44d15b32fbf6bb0c", - "symbols.json": "d39267f0d6c8e267b8d4ec04f600012f08e9a9b3344ea4cbe5ebe8303e5fdd30", - "vex.openvex.json": "28a1e5373cd7d22a0078e4655ca51ac127a81a056cb1bc751add88cbba886b78" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "8030095b4fc7157d09af6fd16fd3fccfb013f5a744c7e13e1bba1fb01201b2e6", + "sbom.cdx.json": "109a4ef5481c4597a26f3172e5f5fd1ead491b55f19c84bb93a46bd6e5c47b28", + "sbom.spdx.json": "619548fa26467f19ddef9a2b1adae3c0fec5b166a3a4f494901ae23ddac0156d", + "symbols.json": "4c4a40db721f39e3bd06a5dd63c408ebf6f8bd9dd3faf1892b2f0a712b81ad8c", + "vex.openvex.json": "13e69a076e5d4c622d82b042ce26129e0fcdf62eb8a800303a23ab9915938c2e" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/unreachable/manifest.json index f9e61c05c..3aab163bd 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-jackson-CVE-2019-12384-polymorphic-deser/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "java-jackson-CVE-2019-12384-polymorphic-deser", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "fab11e90f6e9fab6af782059b6ca8363d39dc57fe11c475566d60aabbdcf5579", - "sbom.cdx.json": "bc802fbfc6cd9cbe431416484872cbbaafe2e1e4bb8d00d1437a8d48bfc2d2cc", - "sbom.spdx.json": "ffb869fd2e4d6d8bbad352d91a431f1038359995701dda2c44d15b32fbf6bb0c", - "symbols.json": "bf928643842abab67d785259cbe9c3b8b655969d037b678bd51e7834fd27926f", - "vex.openvex.json": "d0028bf8a37999413fc66906394072a08ad47b48b53cd914e2e10cf128ad1e31" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "0d7634e488cab16bd206235b80fb635187fe5c648f8ae97f7203d48490209c89", + "sbom.cdx.json": "109a4ef5481c4597a26f3172e5f5fd1ead491b55f19c84bb93a46bd6e5c47b28", + "sbom.spdx.json": "619548fa26467f19ddef9a2b1adae3c0fec5b166a3a4f494901ae23ddac0156d", + "symbols.json": "dc67782d6a011629563b6274b2980b80e60cee3dcb55cab4e4ea9d80dd41046e", + "vex.openvex.json": "c74db782f4df6c74b1a8ec386d2c698bd8ab2f26d7e11f2c4d0d80a5905e35c2" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/reachable/manifest.json index e4f9d7c2c..3d6a7df77 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "java-log4j-CVE-2021-44228-log4shell", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "18e1e5f2d183fced40abed7ffd688b09688f28b0c96b6bf67f3bcd04fd39951e", - "sbom.cdx.json": "372ddb8f9e5d47faaad77ad7c3629eac3283adff306c5891e3fdff073d741e9b", - "sbom.spdx.json": "82f9f8e671138a3d2134be3ce583af42069cd20c2b57657cdad13668b0a9cfe2", - "symbols.json": "69eef48ff4667ba0cd6d454f08405c7fb04891fd8182279fe8d42ced59d15328", - "vex.openvex.json": "b2fad990e76faec9eb20475d1b1a4e0e42012f03db28973237ec338d076d26f9" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "03d8edec093c07c9e0e77b6a52f015095db71ab9b8c2b2fdad245960e40bd2f2", + "sbom.cdx.json": "a43b3ae67d9423a75c709209b5c4c15c389163931bd2c57df1a924f92d0b871e", + "sbom.spdx.json": "b29f8c850043fbb66deb6a8ba9b764a3c66f8527ab47d0ea04cc63f10716334f", + "symbols.json": "b7b75e6116d33e98ae5b92598394095510e27afa8e0facdb617070fd8866d20b", + "vex.openvex.json": "67dd7e3220be878da101bc58d3e55bde4e69a6d56a4e14b4c3c3c5f4f1af8c3a" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/unreachable/manifest.json index ce3c4fb9a..0551aac46 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-log4j-CVE-2021-44228-log4shell/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "java-log4j-CVE-2021-44228-log4shell", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "2bcfc3d70ef20266eeb90a42ad92127be3b7fe17d1a3bc03a8002dc5d42498c9", - "sbom.cdx.json": "372ddb8f9e5d47faaad77ad7c3629eac3283adff306c5891e3fdff073d741e9b", - "sbom.spdx.json": "82f9f8e671138a3d2134be3ce583af42069cd20c2b57657cdad13668b0a9cfe2", - "symbols.json": "32146204dbcc7be27e80cf6a12d15f53b5c2f36a043adad94266f70b371fbc6e", - "vex.openvex.json": "e9db528da741ec09b92b6166f1382bfc340f1effc93a4ad61d0da8817a1df5d0" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "80427fa6cc873a3f440db5686d134709d34613394762ed8dc411dbfdeadaa8c9", + "sbom.cdx.json": "a43b3ae67d9423a75c709209b5c4c15c389163931bd2c57df1a924f92d0b871e", + "sbom.spdx.json": "b29f8c850043fbb66deb6a8ba9b764a3c66f8527ab47d0ea04cc63f10716334f", + "symbols.json": "7e4e19ff912bff2a72dd34cb814b2fd52b63f6dceb7e423ed2eb35a739d6719b", + "vex.openvex.json": "e65779c3e3469b618c2b2c978a66e077e6c70311434fe2ca1364bf30c8b9570e" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/reachable/manifest.json index e360e9491..1c2b227a1 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "java-spring-CVE-2022-22965-spring4shell", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "4e42e16ec75b44cd13fec0687b5baeae4fe9d3e909b2cd420d869831332b555b", - "sbom.cdx.json": "607256a4708ac35a12f7fa3b229158e28bd4ac1183e81327c2369fa9cff4d214", - "sbom.spdx.json": "cf583ebd3bba7aa0dba50b435adcbfdc8878e9c516f6fd540fcf59f16cb989b0", - "symbols.json": "7eb263746a5371dcd706ae6f92ba9b01f16de87ba385d1c7ce5c94784ed309a0", - "vex.openvex.json": "84af60858e98b8b8367c4a747176319357abd5e49cf4826454ee076ae07c759f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "316f1bc49235fad6e8aeb59c95028e79801d1e0e87599dc87cbeb919e55a332d", + "sbom.cdx.json": "05d75b98871eb73a5f81774ce2eb9a74cd36e2e6751aebd28df64993a538501b", + "sbom.spdx.json": "97a3f8f8c8424f7caf000dcf8da67fd12ce7662302f5113d39058f4fba8d7061", + "symbols.json": "c45532d8f5df11d1ba108ee3203b66dc6eef453f7fad1df7b4f120c3be28d8e2", + "vex.openvex.json": "3faeae83e4427b7ad268d55b38d246982713cb18d1dbbb1af7f55bfdec2c528c" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/unreachable/manifest.json index 613f06767..4fc944c77 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/java-spring-CVE-2022-22965-spring4shell/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "java-spring-CVE-2022-22965-spring4shell", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "1ad19ebfd6a160fd4c1290aebaeda41c7dc2850c2355f407b1beb62743e6d385", - "sbom.cdx.json": "607256a4708ac35a12f7fa3b229158e28bd4ac1183e81327c2369fa9cff4d214", - "sbom.spdx.json": "cf583ebd3bba7aa0dba50b435adcbfdc8878e9c516f6fd540fcf59f16cb989b0", - "symbols.json": "18f2e58edb1bb5e9c9526cde7e03f0e0f5c6dcac248b65efb7f4c53272f42376", - "vex.openvex.json": "d214434c583ec25326745612135b6bde54f96c31040b4d20c1e82ae592c38e9f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "d5dfb70311cdfcbb9d9dc00f2c432e21994a567b73afc2e5e51105dd75098a9d", + "sbom.cdx.json": "05d75b98871eb73a5f81774ce2eb9a74cd36e2e6751aebd28df64993a538501b", + "sbom.spdx.json": "97a3f8f8c8424f7caf000dcf8da67fd12ce7662302f5113d39058f4fba8d7061", + "symbols.json": "24c8f838eca93f887822a0e27608d21695c7e77aa5ffcb4f0b7f67e0c7f9254f", + "vex.openvex.json": "64c8b4fbc6462876ab6861b9235c1f11200a881392c55c40371786f5d21fcec5" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/reachable/manifest.json index e177a246f..3c263ddf3 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "linux-cgroups-CVE-2022-0492-release_agent", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "1b1d298f981935c318b3c010219ab24a4446b0525ad1016e914202922bc1661e", - "sbom.cdx.json": "4490a3b78cbac995e5c4ed2e13e948d7e6b1ed042ab693e5d881871282b5d2b8", - "sbom.spdx.json": "ee4568feb4a4c83d6cc10518bbc83a6f985b74a43f2ce4c8e3aaa519bb842e5c", - "symbols.json": "ea98caf1fd5a099bd197eeccb973b951eefd48dbb1c8f1dba4230fb71a71ad8f", - "vex.openvex.json": "cc7ac5157ccc6fff563a060e7af71150eba5af7023e159964fbd4f33629d2d40" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "8b212a35b6bbd0eebf58c888fa3ba2f15df2c223f46aa0cbe3a819eb0b00a04e", + "sbom.cdx.json": "011435c08b0937a16783c5513a7a6997562db09e5683663b72eef0582b117928", + "sbom.spdx.json": "2ffd0b73f7fac20f929aa782ac97496b693846c63cea70b22ca1ab07801dd8e1", + "symbols.json": "c8221bd84c11929566d8460068cc87b5b17fad5be3744b11bfde2f6c66ebb2cb", + "vex.openvex.json": "ec5738e266b360a5b176af280a68c9e147bdfc21a30c6429845d320ff7766819" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/unreachable/manifest.json index d77959e87..6572e86e0 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/linux-cgroups-CVE-2022-0492-release_agent/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "linux-cgroups-CVE-2022-0492-release_agent", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "d12f50088d87da6735bf77315520c649b7b270926487b35427eabf9c09c70f53", - "sbom.cdx.json": "4490a3b78cbac995e5c4ed2e13e948d7e6b1ed042ab693e5d881871282b5d2b8", - "sbom.spdx.json": "ee4568feb4a4c83d6cc10518bbc83a6f985b74a43f2ce4c8e3aaa519bb842e5c", - "symbols.json": "74a8557624a8459c59919421209e564b8592a0a1e1e3c80f9c9ba267140b024c", - "vex.openvex.json": "1f205cca143401fcfcd9c61f33ffce9bf51839546a8031f583ca04aa83ae6df7" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "ceb7e2d85e6a23cc60caf2eb46e5e05cdc8af24661ffcc9ac674ed12234529e7", + "sbom.cdx.json": "011435c08b0937a16783c5513a7a6997562db09e5683663b72eef0582b117928", + "sbom.spdx.json": "2ffd0b73f7fac20f929aa782ac97496b693846c63cea70b22ca1ab07801dd8e1", + "symbols.json": "89e6fe61fa90b366b00e0e7f61bd9f4452e490e6197ea6d606751caa2e31bbb5", + "vex.openvex.json": "3e10a7fdece86c0aa73c1d8a86d693a75ad020d2351458878231944b9e4ae28a" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/reachable/manifest.json index cbe56b3f6..311e0dfef 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "node-express-middleware-order-auth-bypass", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "b105d1e625d001e8e801b67f40b72f13ffaa3889505353e9ae9aa6316ae92e06", - "sbom.cdx.json": "96830b648c0267ae5fea1bc534c2868778028f04b15365c8e42fcf0b9ad8ec90", - "sbom.spdx.json": "432b218689fb7e6affe40229822ed1632823f59ffa7ce19d44ece9a512c4e7a4", - "symbols.json": "5f2abc128a64ae085cffaca5b1c9d21cd22abcac35f3b79963c213a5e965d800", - "vex.openvex.json": "4ae403a8c67fb67d430059af05c72358356f291c13fc3219b97c29d74acebc9a" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "3ec9dce86031af5a893667834e8fd21c276a5afb1156544e0208a58e65f99841", + "sbom.cdx.json": "104dd5cb4497b83d59c6cb0a3e59af02d4f2b52ffa4709086a7dcccb5ef4d7b8", + "sbom.spdx.json": "3f4850fc7da4fde7f97d33d0c6b78a0e50bac716fbb4f0dab2b6a3c29fe302be", + "symbols.json": "8cc43736be4fddfbd8947e03263cc1a3d7301aa4be6bad1d6bf99d91787c14ab", + "vex.openvex.json": "d165dbc8f75c38b68a154f2ad365d686cb327883c96cd88669f4f163407598dd" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/unreachable/manifest.json index ef9adc347..65ad51b36 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-express-middleware-order-auth-bypass/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "node-express-middleware-order-auth-bypass", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "ae08f45e5718acefc99d0146143b699022486b8baad91d32622345b82aca46d6", - "sbom.cdx.json": "96830b648c0267ae5fea1bc534c2868778028f04b15365c8e42fcf0b9ad8ec90", - "sbom.spdx.json": "432b218689fb7e6affe40229822ed1632823f59ffa7ce19d44ece9a512c4e7a4", - "symbols.json": "18062cd7e4c9e4e4313223533ca6308bb3d5d469feca8689aaf897f95830b4b7", - "vex.openvex.json": "25a9db016befed97dd385225ad9bcdd25e7a60d65f049051884e504100ae6d6d" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "7fc0d7bf7870b42b4d216d2c9001446761aa86b073da551776350cbb481b14ce", + "sbom.cdx.json": "104dd5cb4497b83d59c6cb0a3e59af02d4f2b52ffa4709086a7dcccb5ef4d7b8", + "sbom.spdx.json": "3f4850fc7da4fde7f97d33d0c6b78a0e50bac716fbb4f0dab2b6a3c29fe302be", + "symbols.json": "45aa8a689a6fcca0a0c96e587da654d30301b37190d70dd25240231e14cf4df2", + "vex.openvex.json": "3fa11fea858bb9520c1b9c656d1d6b8191fb15a11aa92ccb933ce999b115a29b" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/reachable/manifest.json index 226e9696f..c5a69cd3c 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "node-tar-CVE-2021-37713-path-traversal", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "f4e47da91b002b9bbab2e79fc84b103ec3310fead248588abfe77c4bc0b9d8eb", - "sbom.cdx.json": "3109ba08a7a9a908282f3d2014e78ff74418b925ead00b64c26400c508cce9d6", - "sbom.spdx.json": "3422be00cedf9ff4a4a808bfb7c4abdd491fd545da2a6076cae6096710c4abc5", - "symbols.json": "d9b88cab4fdc99df1f009fd65aa70da74c4f45e79919b754092d63439f1cdf9e", - "vex.openvex.json": "f9e650b96ea369d484ffecb37787ce43d54feb09c2f49e2bf948213be7855c9f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "0bbbab7a034667021473bd75c43b5f4317e5b99aa55a1ffd37696c61899ffe14", + "sbom.cdx.json": "bd237786ad3208f9f41ad2b56d05c4f3482966628f28bd7ece00dc37d247fb3d", + "sbom.spdx.json": "971e3ef7be1edbf5b58b72753740742773333003d953ffbcc88581c97aea9464", + "symbols.json": "c532dbbb307244b4f83dab9b7a767906c90e4bea518f3753159064e34d4d70aa", + "vex.openvex.json": "bcd7c056e063ad8ed87cdfdfd3bb4e9bff1753acc738380b2e6c779db6f6ce46" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/unreachable/manifest.json index 8d229f696..fb0b262d4 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/node-tar-CVE-2021-37713-path-traversal/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "node-tar-CVE-2021-37713-path-traversal", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "d4c0bdae4cbc822f587e626fe1dab278fb486b295f0b4451475afdede3bccb18", - "sbom.cdx.json": "3109ba08a7a9a908282f3d2014e78ff74418b925ead00b64c26400c508cce9d6", - "sbom.spdx.json": "3422be00cedf9ff4a4a808bfb7c4abdd491fd545da2a6076cae6096710c4abc5", - "symbols.json": "31863df6aeb0fa6204816c0a4efc842b0af7ce8db22e9cd15849f94aa27b7e87", - "vex.openvex.json": "204c982c4db778940bc3a65dec3c8b5e6fb867e5f348c6e5dc28be0f643fd01c" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "02751e4826f1ebec26f76961b3993f0bf33d3af8d1778fb0ae384ef890eecc5c", + "sbom.cdx.json": "bd237786ad3208f9f41ad2b56d05c4f3482966628f28bd7ece00dc37d247fb3d", + "sbom.spdx.json": "971e3ef7be1edbf5b58b72753740742773333003d953ffbcc88581c97aea9464", + "symbols.json": "806a418424cbf187306971605d13cc4243e9203b8e0529eebbc9846ed67314b1", + "vex.openvex.json": "70174199bce72123d6a646dce6508d6693d7da1a92b464707b6a3fb3b2e4db7d" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/reachable/manifest.json index 7e5077237..518bff2a6 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "openssh-CVE-2024-6387-regreSSHion", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "4f4b92f6d94f293e8c200a9254b88a92d5777cfe33a5b4e52ad5953284af6c24", - "sbom.cdx.json": "d203047eb82f5fe63bc1254c83840dcefd25c80caa29d209416a4363b216a7e8", - "sbom.spdx.json": "d6be2733c73b4615707f8b81b60b0c266febcd621094b2e75448073269aaf8dc", - "symbols.json": "88fd0e3ec2b0f54c2f8bc610fe6a5086a361798f2d1a6e52bb21a3852bf1953e", - "vex.openvex.json": "913cf9478b644202660c348f59ae09b540529549cc6937047a29d8397a43ff6f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "573a54be180f06ac67ad206a9fc55b6e24a92b5560d931ecef7e534d35e0bd59", + "sbom.cdx.json": "04d9991ac2950015546093ad479344b1ab8365495c54a45a49ce6738d115b13d", + "sbom.spdx.json": "adb3128162032496f058f46b0e821b4f8c1a673c8ebdcd1ba3b0961912c95886", + "symbols.json": "73bdbf7929a114b682f37794706cbeb86d998a5558849fb17a6f74e07ddec575", + "vex.openvex.json": "a9a5faa5120965062783d59139da86fb1e56dfb946e033678ce908889a65adec" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/unreachable/manifest.json index 857d1cae9..a11210ca6 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssh-CVE-2024-6387-regreSSHion/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "openssh-CVE-2024-6387-regreSSHion", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "d5dd3c0bf3efa2aa38c5453fedf2a541a350c32f0b3f48507e9077b6a8d5674c", - "sbom.cdx.json": "d203047eb82f5fe63bc1254c83840dcefd25c80caa29d209416a4363b216a7e8", - "sbom.spdx.json": "d6be2733c73b4615707f8b81b60b0c266febcd621094b2e75448073269aaf8dc", - "symbols.json": "bfc07a65bf9e823635068e606a75b6fb3cd8bd7a07493432ce3986c6ba22c028", - "vex.openvex.json": "485f80b704b9e4a1f5a6c16d2d89eb02704f5337d47ebbf90cad6372a412bb91" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "e6ecfb707f6ef8b89f52aa4883bc48642dea20a3d7bf647fb04b30a55e5e6be0", + "sbom.cdx.json": "04d9991ac2950015546093ad479344b1ab8365495c54a45a49ce6738d115b13d", + "sbom.spdx.json": "adb3128162032496f058f46b0e821b4f8c1a673c8ebdcd1ba3b0961912c95886", + "symbols.json": "d57f06dcd7f95bf8dcc3c8dc7e2a5096b3a0b36098b9bb7714d4a434dd190371", + "vex.openvex.json": "25a0ed4ff5e7bc23f5b0c80c2264ad14b6a8a1bb124cf32360a227b2b2e68daf" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/reachable/manifest.json index f64cccae8..641cde7fd 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "openssl-CVE-2022-3602-x509-name-constraints", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "38e3f5d0bee340b5e41aae8c3f81a4d8c9f8dbcd9a558a562e072e54f235e11c", - "sbom.cdx.json": "f38df383173720772d3e2da65d3c797c5f835dc6b69a3ec6fd2b3b3d1d108a59", - "sbom.spdx.json": "3658e0973b3a78e9b497e16d86dc73c0d89ec21b6519911ca78e2e8b1a0688a2", - "symbols.json": "49fbb599d179d7a5fc88cf24f6e6e9267f0fae43763c89c959342a795e61334b", - "vex.openvex.json": "b7ce60d2e199ff3d58036665e268edc493114ce82d8e49c26b2e701d390b4198" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "bcc9ce550ea18fae6bd12fe8ff7af87e39b751c1f74735d003598548569858f7", + "sbom.cdx.json": "374cd5f25f0fcd1b58eb23707842ddc95a7755b934a4980ce128d3e03199620a", + "sbom.spdx.json": "0edaca9b8d2b7bceed84e66f0733a4ce66bfecfeccb60ce913f67048df3bb193", + "symbols.json": "a98a37d8759a6e9823d151d3485ef900e455bd6c7c0b47dae47a471ad0b4b8b2", + "vex.openvex.json": "a06ce87aed550880248f6b4e7bd5c78b9a3c967fdda83868557d4cbd2547cd29" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/unreachable/manifest.json index 2652187dc..b2c7f275b 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/openssl-CVE-2022-3602-x509-name-constraints/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "openssl-CVE-2022-3602-x509-name-constraints", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "7a53699425f69e94c99a5956a738d33ff8b213e79ab6c685cd2e5a516145d2a4", - "sbom.cdx.json": "f38df383173720772d3e2da65d3c797c5f835dc6b69a3ec6fd2b3b3d1d108a59", - "sbom.spdx.json": "3658e0973b3a78e9b497e16d86dc73c0d89ec21b6519911ca78e2e8b1a0688a2", - "symbols.json": "7093617c7478ac80d89ac9e887d7ea16442d589a70d3f0b447ba0ef5ccc1a8f8", - "vex.openvex.json": "576085af51d7388e605b6898261dadc425b4407e6182673d55108ff03779a7cb" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "f114155ee62717f1d24fa2bdc42870eedb174e31a125e7e02a0fa2c469b10860", + "sbom.cdx.json": "374cd5f25f0fcd1b58eb23707842ddc95a7755b934a4980ce128d3e03199620a", + "sbom.spdx.json": "0edaca9b8d2b7bceed84e66f0733a4ce66bfecfeccb60ce913f67048df3bb193", + "symbols.json": "0cede4adadb502cfe38e2bfa85fa7886d1bb112e929574de1d7427b512c97b76", + "vex.openvex.json": "e478856a30ec642dfe6b63d8937de0a2ded4f73ad6d161f61b90326fbd6b2b65" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/reachable/manifest.json index 390074dc5..0d2a04d89 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "php-phpmailer-CVE-2016-10033-rce", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "6b470ec13b3e8db799ecccce16b10d36d6655e5951038801b51955071ff7ec90", - "sbom.cdx.json": "f8d1582c3b6478cbd4879bcc07d60b21ff394df14f616bf12bb269217cfb57f1", - "sbom.spdx.json": "553220191e8f1fa9956f47ebd232ee2554e531b1928ee9d3e1d479b5b360139d", - "symbols.json": "f86de9fa107355075ccc3407dbefa15c27514975c211a9017e19ccc0cada9990", - "vex.openvex.json": "c0117f2b546df8ebed8409ea7472ea8a8f3f959f2b17d44c738a7cd24a209d78" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "8a4c1f78c866a351322eb9b12dca1b0a6218aee5094bca0a4b7090e00ae524cd", + "sbom.cdx.json": "9fc0ad284188e41a23fc678128e5e0fa263c39431e7c976c82d8ec7d0b6b0339", + "sbom.spdx.json": "96cf94ee5085078d14ee5c19666a9e146c278b785ed57eb2b47faf45b9d18b85", + "symbols.json": "2c47399bcb375356772a6f5fd4e1230721a0807f450b33dd9a512e72f0f932b0", + "vex.openvex.json": "2b258cf5cfb4a08edabcc0d865c4c4531b67a59b6c3835412f4b417e36693f84" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/unreachable/manifest.json index b21f49441..ed3d315ae 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/php-phpmailer-CVE-2016-10033-rce/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "php-phpmailer-CVE-2016-10033-rce", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "7d3476c3d86e0dbef128efdfc16a88bbb13e1df0095a17a97a7fc75402da42b6", - "sbom.cdx.json": "f8d1582c3b6478cbd4879bcc07d60b21ff394df14f616bf12bb269217cfb57f1", - "sbom.spdx.json": "553220191e8f1fa9956f47ebd232ee2554e531b1928ee9d3e1d479b5b360139d", - "symbols.json": "891d0b017f95ff6d3f7f9e06495b39ed53565eab469d879f1a2fad2ca63e632b", - "vex.openvex.json": "e04cbaf855c3d14e918ad06a524d89dabb06c427e4a96e72eaaaa86ff16c5595" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "03c99d5de2d6da9de07480d43bed6ce79a73b1a9abb2ccdd02c04c2eaa3c9bc4", + "sbom.cdx.json": "9fc0ad284188e41a23fc678128e5e0fa263c39431e7c976c82d8ec7d0b6b0339", + "sbom.spdx.json": "96cf94ee5085078d14ee5c19666a9e146c278b785ed57eb2b47faf45b9d18b85", + "symbols.json": "27a70634762c365d15ab5135cc5eb54721ad8407ae295ca71ca227f41847569c", + "vex.openvex.json": "56a227d9bf325b0dce2875c99d09bf999d2c7b17402af641c6902314108ee980" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/reachable/manifest.json index 8f5a74802..804686874 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "python-django-CVE-2019-19844-sqli-like", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "91bdcbbf9400beac3d6ce39b81e5794f5a6715657ff541ed9e3d947b29332096", - "sbom.cdx.json": "12b2d32433bdf9c8c56c1f1de9dbd3d62c0205d815448789caeebe53cc7199cd", - "sbom.spdx.json": "ebb5b23674d0d6b6877e60a658ead722cbdfb2097b3d98d16bde492544334b4d", - "symbols.json": "b450cfbae441529396dbe049f5edb0e5a9c95e805d72c9baff25551a93e5633f", - "vex.openvex.json": "aa8c5da2e3f03d116d045cee53bb1fd52a2d20a42cd935b7ab0649aada9c1eca" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "2ccb39511b35781e96992a480df92beef1c8dbf600d46090b309bfa459397b4f", + "sbom.cdx.json": "33856cb8dfc4b3f14550762c0d6f7d93ed4bc5bc249ed57fe963f7861839bf24", + "sbom.spdx.json": "49ad943b01713c7b711ca2636b351e96581f1323b4be819c3ef25d5cbeeb78c3", + "symbols.json": "b9e2cd285f58d83a44807eceb3011431bab2547dd4f8157f59e685d17b55a384", + "vex.openvex.json": "390fbd7d3099d948046fb31d83e805ea532ce7fb20abdbb270eea55d4c7d3019" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/unreachable/manifest.json index 656afdefb..93dab7e46 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-django-CVE-2019-19844-sqli-like/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "python-django-CVE-2019-19844-sqli-like", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "f8926492d1b5e8a15c9b0f82087797b26d0609bfb13f10d14f7803fe6a129d95", - "sbom.cdx.json": "12b2d32433bdf9c8c56c1f1de9dbd3d62c0205d815448789caeebe53cc7199cd", - "sbom.spdx.json": "ebb5b23674d0d6b6877e60a658ead722cbdfb2097b3d98d16bde492544334b4d", - "symbols.json": "69b7846dd716b8730d2e08a2e293124949fd3642d9fc83837612a4d9d228fad4", - "vex.openvex.json": "bbadc21dc72a1b9dd8e395b2b40c1dac987189a7f95c0064714737b1f9b00758" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "ee0cdda20523335d8c65739d9509a710000e11d8bb2ece93ec7930e8d06590a7", + "sbom.cdx.json": "33856cb8dfc4b3f14550762c0d6f7d93ed4bc5bc249ed57fe963f7861839bf24", + "sbom.spdx.json": "49ad943b01713c7b711ca2636b351e96581f1323b4be819c3ef25d5cbeeb78c3", + "symbols.json": "2f907e2686535d69767522c43fc0c71962ef6ce8bd9e48746707887ce186bf07", + "vex.openvex.json": "a6bfc8b8e86ca4f9cd2d2d107a14531bc13f84261a13b5cafb4e8d4b1c92c01b" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/reachable/manifest.json index ac88811a5..b9ba20889 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "python-jinja2-CVE-2019-10906-template-injection", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "bd19518d4bcaad1b2248ad44087296d0961c0b69ff92dd8dd3e02f2dd7ae50ad", - "sbom.cdx.json": "bee4564ef6e541d6c9342da6e9ec5c32245934af538285742a70b1a5574bab63", - "sbom.spdx.json": "fc8393d30763d114ec156faad13625a01d99637d6d5f94347f9103ec2f128c70", - "symbols.json": "12e9bd031247b0e7ced80b9f0be7366d53e3f6bc55dba8bc8e29dffa33b1428a", - "vex.openvex.json": "5831de3b541eb9cd8e318c3d9c58d4722c3706ae86dfe593edb7052bffa69d59" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "2e56690ed9899e9ddad4a2dd9dd3715fe3b6349cc165fb10b797500d3b7bb240", + "sbom.cdx.json": "8a2681fcd3eb2aa82e2f0380126a9fe2caf130aac6ae4c66cd47f971c4ea347e", + "sbom.spdx.json": "4a04c563f3cf1b8a7b84add3443b9f2372150910844c2160193f26b75c004ff6", + "symbols.json": "97f7be8fae7c41424553821007c4e8ce0784c21014ceba12d77a8487af445ebb", + "vex.openvex.json": "088909aad48426345068b6373a27bacafcaa64fc49ecef43a15326d307f8b2e6" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/unreachable/manifest.json index 6df8ba5f6..02797d268 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-jinja2-CVE-2019-10906-template-injection/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "python-jinja2-CVE-2019-10906-template-injection", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "5e976a5b02be9d80f73279250b9c7dc6a7a35b4949485ec8a3e654b56e92b472", - "sbom.cdx.json": "bee4564ef6e541d6c9342da6e9ec5c32245934af538285742a70b1a5574bab63", - "sbom.spdx.json": "fc8393d30763d114ec156faad13625a01d99637d6d5f94347f9103ec2f128c70", - "symbols.json": "7c9c852ec05f723da16662dafbee27d7f30930565483b3efd9a98124daf965ce", - "vex.openvex.json": "4814babf16abd8827888413cb3545fa592a544da2818c510f39f5bf6d626fcce" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "df69cb0fd97c3f90700905edbb739c872b13bca26ec8b23b5fca7df95c88e649", + "sbom.cdx.json": "8a2681fcd3eb2aa82e2f0380126a9fe2caf130aac6ae4c66cd47f971c4ea347e", + "sbom.spdx.json": "4a04c563f3cf1b8a7b84add3443b9f2372150910844c2160193f26b75c004ff6", + "symbols.json": "e8b79c2d1c222102e4dd1b3f009c98301bb9d20bfe535959a968719f55dbe558", + "vex.openvex.json": "14189de5cdec146e3f3690f9a33bf7bd43e788c1bd52deb9fccfbddf548d0fb3" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/reachable/manifest.json index a81e179ab..bc29678c9 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "python-urllib3-dos-regex-TBD", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "b76721e7be720aab8e6a391c4307a49c9ac5f91370d5ad94e6990851cd9f5599", - "sbom.cdx.json": "92f85cd1caf5e0d40bc489c74f61a10ef452f8a5929e95a52436e0a67827f1db", - "sbom.spdx.json": "14a0b0f27eddab4452da4b7d9da1992b8eaee4c9a56720accedae3f5c1669a72", - "symbols.json": "03872ddb4ff47802e1ca998b01e2578877bb0c73c0584d804002082f0be2d22f", - "vex.openvex.json": "c1b2c2b8af62f5d198e8986f25763c64126b7c930cca6580bb115351080478f4" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "77b65a72e7061171dd9bbabb55260c005e45c71156349c68995f5da21249f01d", + "sbom.cdx.json": "6de2dac2a942c4f98be45913bc283490e0a633d96f622864eba2f7e9ed40ddef", + "sbom.spdx.json": "fcc1da998d896c2a8d6c0b0386ae5a492ae242cc83dc03daaf2b6ee55d8ba9bb", + "symbols.json": "0de9697f4fe6f5d80df4aec4593599f6dbfbf9c92f2e19e4e8f6d39630a37aee", + "vex.openvex.json": "c785b009bc7c625f1e3cda129ab45ac436b43dc726f3902d092bfb4665a5a1dd" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/unreachable/manifest.json index d5e30a9bc..14631f381 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/python-urllib3-dos-regex-TBD/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "python-urllib3-dos-regex-TBD", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "022d9b1a05425205e685a3c3e0f4991f3dd0e311b0b443b0cbaed3e538fdece1", - "sbom.cdx.json": "92f85cd1caf5e0d40bc489c74f61a10ef452f8a5929e95a52436e0a67827f1db", - "sbom.spdx.json": "14a0b0f27eddab4452da4b7d9da1992b8eaee4c9a56720accedae3f5c1669a72", - "symbols.json": "2b7cf80eb18cbc9d815a5d9310b167b7bdebd5ee3288d679af1a60aa39bc2efa", - "vex.openvex.json": "b57bc50d2d4d3075d4797ed884546d733939c54b7ae5cc4dcf3e8aef1b5def08" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "fee28d40dd848e5e59b662622f33e2644e4328c6a2eb1a4f22f558fea0c69dfd", + "sbom.cdx.json": "6de2dac2a942c4f98be45913bc283490e0a633d96f622864eba2f7e9ed40ddef", + "sbom.spdx.json": "fcc1da998d896c2a8d6c0b0386ae5a492ae242cc83dc03daaf2b6ee55d8ba9bb", + "symbols.json": "bafb8c6703ba42f7fcb2d1bc5bba702282012d10f7d7026729083761e8b6bf26", + "vex.openvex.json": "342a13c0f33bbf5228756e7444aa1a0740b0f971115ead4db2668669e8055fb5" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/reachable/manifest.json index 63c428558..69a170551 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "rails-CVE-2019-5418-file-content-disclosure", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "a1e201f50ab1de6dc8a63a368bd11b8a684f68232fc3afd9c159303023f93dd3", - "sbom.cdx.json": "e311708a3d964928577bbcaa5422955d01d4ff41276cb1ced140d291c432f5df", - "sbom.spdx.json": "5ce3a3ad477927485ddc6e4a4600c780336dc086a4ec931f0be0472858deb88a", - "symbols.json": "bad5fe3ba1b5ebed91c92b95a4a579459220eefbda301f14f4c6e29b196fc646", - "vex.openvex.json": "335d81c4776df1c000846591e5e34c97c5e9fa5dc0da140524cc68a697d95476" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "c9d31abd8f660694b9cc88b0663390c36be5ca0c16da8061911ede2af396d64a", + "sbom.cdx.json": "c50edcd3ebed1a651e29f3ab41cdb37e2a6a1f7bdb567191bcf83fc1e76eba24", + "sbom.spdx.json": "7bcda4289b9cd770cad6408cbb1e9bbd6b8ef7ba15b79b795b5a183f22722925", + "symbols.json": "895ed278a0c2eb90a697755ff7509d7e9df3a1aa26153d2deea2e2e858a62aea", + "vex.openvex.json": "500bbf7564559d0f10e4cdf97f8142868d6d068b72bb223b8a4f6850e917aa93" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/unreachable/manifest.json index 7c81f59c8..7c718aba9 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rails-CVE-2019-5418-file-content-disclosure/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "rails-CVE-2019-5418-file-content-disclosure", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "bdc4a9b0600b2adbe376631ff42c62b0d2c09913ee5e6b709cfe2274aeafe5d9", - "sbom.cdx.json": "e311708a3d964928577bbcaa5422955d01d4ff41276cb1ced140d291c432f5df", - "sbom.spdx.json": "5ce3a3ad477927485ddc6e4a4600c780336dc086a4ec931f0be0472858deb88a", - "symbols.json": "6bb75ea71e5e1bfa9fd3a86b40e47ce436fd0b2d0d542657858c01dab6b818fa", - "vex.openvex.json": "d2958b20176758ba49b37add31f335c152b91f17aeedc2b0407bd2d673f7580f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "12a1d28dc31ce8a3e381fbfb195d942c879ac31003c2412d40d8b6e8a3808318", + "sbom.cdx.json": "c50edcd3ebed1a651e29f3ab41cdb37e2a6a1f7bdb567191bcf83fc1e76eba24", + "sbom.spdx.json": "7bcda4289b9cd770cad6408cbb1e9bbd6b8ef7ba15b79b795b5a183f22722925", + "symbols.json": "ae28dfed9d506cd92a0608f8a742716764b7f3f15d7b35b5f6990010a1d0b8fd", + "vex.openvex.json": "f3fe8061e72d74532921a4ac21107bfa5121cca2ce011c38580438738c071174" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/reachable/manifest.json index 1a2ee75ec..a0104f01e 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "redis-CVE-2022-0543-lua-sandbox-escape", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "f8c53484a0f0b476c46fdcb74eb369e9043528019ee36b0537be0964d1bc443a", - "sbom.cdx.json": "262ee8cfcec9893d5eef21da86a51563d2f905fd0ded264ecd9b295c20ad4ff5", - "sbom.spdx.json": "207e2ddd259d5cf0d1b4b6e26bc4abd26d38e119dc4c635b0600d5ce4e4d4fd1", - "symbols.json": "add91063b52b65788cfcb6925811a9e51031d7d7bb7c97fcd4e45e1b1c702e43", - "vex.openvex.json": "b4e4b865b0982c89f3d8bbe27d6984016a9b43b7282b9c158063c65a67c4dece" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "fbdd8b9e479d40cea9068a83c98619af1aa18dc3abcafc2ea2bdb463c25710a9", + "sbom.cdx.json": "66bcb9f575207e62f46230e9056c229d07821d700a7b90ebb6f84baaa28bc7ea", + "sbom.spdx.json": "02a37695184dfe333892c420b4890ea69497e2808aeeda42c6c5e211919a5db2", + "symbols.json": "9a5d1611a6e4d6d38feaa591be880bbb157680828b6e7c756442bc6995d960e7", + "vex.openvex.json": "1d86a9a2973f1ab8e5f57c57857c49da3d0d02aa54a8b2d5425021d2f9627690" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/unreachable/manifest.json index e8cc41775..9900a1b84 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/redis-CVE-2022-0543-lua-sandbox-escape/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "redis-CVE-2022-0543-lua-sandbox-escape", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "749316cd4d32d0fb0f9744f4ae7592674daed71c9000cd8b8e57f0372b665902", - "sbom.cdx.json": "262ee8cfcec9893d5eef21da86a51563d2f905fd0ded264ecd9b295c20ad4ff5", - "sbom.spdx.json": "207e2ddd259d5cf0d1b4b6e26bc4abd26d38e119dc4c635b0600d5ce4e4d4fd1", - "symbols.json": "51e50884d378b4f49dc742323912f3af0d600c120e1f448ff3c6929cd265ecff", - "vex.openvex.json": "7e8204c4e3ce4ac26028a390c3fecdd1251d6ad6e16653b1f972f2dc30688c4d" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "bab266ab92040c3977c2f17cbddb6e91e0e9ea748b720ba4b3e35372063c8c75", + "sbom.cdx.json": "66bcb9f575207e62f46230e9056c229d07821d700a7b90ebb6f84baaa28bc7ea", + "sbom.spdx.json": "02a37695184dfe333892c420b4890ea69497e2808aeeda42c6c5e211919a5db2", + "symbols.json": "e1f5a9f63042d050a3966e95a5902797e00ae1703a8ecb69dff149e1bf8371e8", + "vex.openvex.json": "d0beebfb1d7a3cb086633040ddda7f9d4bb536d0df4176e8c093599988bb75f7" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/reachable/manifest.json index adff20632..98c8a2964 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "runc-CVE-2024-21626-symlink-breakout", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "7462c1ff3fcfeaaf75793bc8cc6d433cfddcfc3c01aeed80d808368dbe7b208a", - "sbom.cdx.json": "102a4aa9b079e95ce6603f1077a03716215067f9db450f6eeeee3ffe91a3c03f", - "sbom.spdx.json": "84480ee0f19dd5bc8379dc257f9e86cec0644131f71304efab9d6f9343e5ddf7", - "symbols.json": "b3e49a7b256a4185969742db54584eaaab094893b77b155c482e5784123d4705", - "vex.openvex.json": "730b927596ceff24604ed3fdc008179b3b6a524e8ba1b2ce248706dfcbdd8e9f" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "94d2522b2ab3632adadffdf2ca8c991260d2bbd1690fb4a00beb6207b1ba5c49", + "sbom.cdx.json": "f5d25c84c10d3588526ba08d1a03a8105f8da1279a44e0defee66285814437d4", + "sbom.spdx.json": "ee5aaaf68271588ee2b33d04e3815ca9a3f89a557a55c1f8d917c2af1b813c16", + "symbols.json": "1b8a40f0c8aabd9f84f06647490f450170c48c9cbba929a50caf441c92791df8", + "vex.openvex.json": "ca87293d1831169e427182e37e52713495b9e78a7e7b14f174012867f3cff6b9" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/unreachable/manifest.json index 4eff25a06..a2d07d227 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/runc-CVE-2024-21626-symlink-breakout/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "runc-CVE-2024-21626-symlink-breakout", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "d7aa1571a9338d19994f22672a0f21805b91433c44548db710f03bec7cf19293", - "sbom.cdx.json": "102a4aa9b079e95ce6603f1077a03716215067f9db450f6eeeee3ffe91a3c03f", - "sbom.spdx.json": "84480ee0f19dd5bc8379dc257f9e86cec0644131f71304efab9d6f9343e5ddf7", - "symbols.json": "445405207e280de83a29d0d9eb24d73dcf0b822eddbf6af7b4fbf8e5f9f61d51", - "vex.openvex.json": "4b5e515c8df7566a85a9a5628558157ce4f26d271b3c9d1ca9775b44970d9813" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "2f4803eb568d4f294f4241d068266b6241fd20f463e6945e1df3b84ccd89459e", + "sbom.cdx.json": "f5d25c84c10d3588526ba08d1a03a8105f8da1279a44e0defee66285814437d4", + "sbom.spdx.json": "ee5aaaf68271588ee2b33d04e3815ca9a3f89a557a55c1f8d917c2af1b813c16", + "symbols.json": "b48973486b29e184ae09ecfb9264a400fefae5c2df4d0d5f4bd868453fae1f99", + "vex.openvex.json": "06610030b92a3a5eb4e77c44c87066ce66dfc0018e4477de99f4bbf70424cf5a" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/reachable/manifest.json index 8b87a500b..e54915dc6 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "rust-axum-header-parsing-TBD", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "db51a2ce78f32b9cb4ec3ed5553129592d6b283269b781da07c2b9d248d3eded", - "sbom.cdx.json": "d591991aad584887e1febb7158aee2a8d255cb3bd904647c0465b2517f9b0420", - "sbom.spdx.json": "952f2911841aef80876fa5f3ef6ea3bb025d3fdf15af12a8a2dfd1a4912de327", - "symbols.json": "91f2bb239888454d2e868267474d31c7849348efab10296d50c2d542be287f18", - "vex.openvex.json": "78675798250d81e833b06e465b80e367cc9a3385dc83e26c9126dc8b81e39f07" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "f34d41548950529728b47d39699260a5a3b496f5b729c7981ef2d70622136df9", + "sbom.cdx.json": "ff9bfeeef7e41d934a051d5c4e20965819d2c4be0ff9ad68ba250eccae3aa487", + "sbom.spdx.json": "1de691c4665d49162633b6571bd142fbfdcf79a0c8bdfb6bbf8f8d4783587d01", + "symbols.json": "fc3923137f963fe08398a0cfc11d51d063104758ea574705476bb5fb07b0d6e0", + "vex.openvex.json": "f0aa98d011f0012ff230c44f69aaed51847a4ad9930bac52aa4467405c2122f5" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/unreachable/manifest.json index f3c4ea50f..6d7a0fafa 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/rust-axum-header-parsing-TBD/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "rust-axum-header-parsing-TBD", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "6586881d4ab2e395e45ca6ca2c018ec6c8ef349d08a2d7729b1c2a39b79b578c", - "sbom.cdx.json": "d591991aad584887e1febb7158aee2a8d255cb3bd904647c0465b2517f9b0420", - "sbom.spdx.json": "952f2911841aef80876fa5f3ef6ea3bb025d3fdf15af12a8a2dfd1a4912de327", - "symbols.json": "5fb1a7d62e2b9c38c22cd55bb48a71a46c38b7d9aac5eb55064ca0918f246e7f", - "vex.openvex.json": "e24870c312cf61ce8e377993e2acda9f36ce817b8f97eedb339713cca52d8877" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "918f742dbfd9e786640660f13f60b1f9da329caee367851990c7d5b678fa5c8e", + "sbom.cdx.json": "ff9bfeeef7e41d934a051d5c4e20965819d2c4be0ff9ad68ba250eccae3aa487", + "sbom.spdx.json": "1de691c4665d49162633b6571bd142fbfdcf79a0c8bdfb6bbf8f8d4783587d01", + "symbols.json": "4ff5b34b01575558256364c017f6e3ed4dcb9c6d077b732d2dff1936431f607b", + "vex.openvex.json": "48e178a71126b1c57aaedf47ed85da10a8b391ddcff61e118f7fc26b1786e490" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/reachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/reachable/manifest.json index 2095e2a24..9ef2226e1 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/reachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/reachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "wordpress-core-CVE-2022-21661-sqli", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "3400d2e5bb7e2444c5d8794deaa470162ae84ead4da2f357578bad79ff79fc46", - "sbom.cdx.json": "18ee89df532562af951ae92170a0c6fea0cf5260275becf2b136f58929565dd8", - "sbom.spdx.json": "d8d65f2816de794aeda9b758f9b85d5dc2e771179b1c9a489c83bb9098088eae", - "symbols.json": "b13fee6e67fd09b2655da862bc9557f516a73f8761d3d3ed6ac81839e9d61411", - "vex.openvex.json": "edcaa5bd78afa9a5c2254a536efb9413bf212b247fb81b83d39c71d1a63aff49" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "2491f25c8faaa3fcd6be0af96ee0bdb047ee457c674a3b0533a07bdf4f7bc9e6", + "sbom.cdx.json": "4d680dc644aedae0656a9aaf619804cc5db818071b15a4d4bffba85e4a72ec16", + "sbom.spdx.json": "654a9c21de6aece294f38e1b6590e82ddd4bbe92ca8dc17b9cdf404f7f423a05", + "symbols.json": "5e50e1037f4c8d80ad3d4e589a62eb4748e37410e4dd1336b9556bcebdb7f2fa", + "vex.openvex.json": "789eba2e95cc32972356f777b9b314cdd84d7ab8f62f9c65599ad46d53c1171c" }, "schema_version": "reachbench.manifest/v1", "variant": "reachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/unreachable/manifest.json b/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/unreachable/manifest.json index 1a0fe8565..8625eaac0 100644 --- a/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/unreachable/manifest.json +++ b/tests/reachability/fixtures/reachbench-2025-expanded/cases/wordpress-core-CVE-2022-21661-sqli/images/unreachable/manifest.json @@ -1,14 +1,14 @@ { "case_id": "wordpress-core-CVE-2022-21661-sqli", "files": { - "attestation.dsse.json": "30937c552127484d65b409f39fe31caa5a4f071142892cc336be2a22d747331d", - "callgraph.framework.json": "9ca4a75a6d19744789c7b34011e67126ab749b9626158ae57b03dbdfa6147a5e", - "callgraph.static.json": "0ae8872a65a499cc8109b36eac53dfa0b6cf60b29482ddcdab6992c49e35ec74", - "reachgraph.truth.json": "a84be394e3a8acc773ba5ffb2a6408850a1cc6e0729ed9223010e117d2270f8e", - "sbom.cdx.json": "18ee89df532562af951ae92170a0c6fea0cf5260275becf2b136f58929565dd8", - "sbom.spdx.json": "d8d65f2816de794aeda9b758f9b85d5dc2e771179b1c9a489c83bb9098088eae", - "symbols.json": "3a8ad137ba4c6f6701f2f974c5bf44ff8e4ff098fdf505f2b874d5b24a62f6e7", - "vex.openvex.json": "8ed2bbdb19741b03709d7bd095cb2ee8f2c035aa736cf1cc8938375a60f9f0dc" + "attestation.dsse.json": "12ced21ccc633b0f458df44e276c954ccdbb14c5acd0d234fdf7934eec48696f", + "callgraph.framework.json": "86ebf343e4b684a3bf2b3200e0bd1849397ea69f280330b1095aceefdff799ce", + "callgraph.static.json": "99c850cccba6641635d1c668f831c80667930ddcd1f7acb2fe9c4c7771c63e7e", + "reachgraph.truth.json": "0855e5a1023d4a03c71ca526cd383cb09adb3ab67aa91039c8b96bb370aff3e5", + "sbom.cdx.json": "4d680dc644aedae0656a9aaf619804cc5db818071b15a4d4bffba85e4a72ec16", + "sbom.spdx.json": "654a9c21de6aece294f38e1b6590e82ddd4bbe92ca8dc17b9cdf404f7f423a05", + "symbols.json": "52cd52c683750cccde96c6d0034c129ede7e030bdf5df7b21d1b9bf64eb3b280", + "vex.openvex.json": "72c17746b337df751658ad7104d5f4e5962d97f9227ecbab810e7d2d8dbcad96" }, "schema_version": "reachbench.manifest/v1", "variant": "unreachable" diff --git a/tests/reachability/fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py b/tests/reachability/fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py new file mode 100644 index 000000000..be7efa4c4 --- /dev/null +++ b/tests/reachability/fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +REQUIRED_FILES: tuple[str, ...] = ( + "attestation.dsse.json", + "callgraph.framework.json", + "callgraph.static.json", + "reachgraph.truth.json", + "sbom.cdx.json", + "sbom.spdx.json", + "symbols.json", + "vex.openvex.json", +) + + +def _sha256_hex(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _locate_repo_root(start: Path) -> Path: + current = start.resolve() + while True: + if (current / "Directory.Build.props").is_file(): + return current + if current.parent == current: + raise RuntimeError("Cannot locate repo root (missing Directory.Build.props).") + current = current.parent + + +def _update_manifest(variant_dir: Path) -> bool: + manifest_path = variant_dir / "manifest.json" + if not manifest_path.is_file(): + return False + + with manifest_path.open("r", encoding="utf-8") as handle: + manifest = json.load(handle) + + files: dict[str, str] = dict(manifest.get("files") or {}) + + for required in REQUIRED_FILES: + required_path = variant_dir / required + if not required_path.is_file(): + raise FileNotFoundError(f"Missing required fixture file: {required_path}") + + files[required] = _sha256_hex(required_path) + + manifest["files"] = files + + with manifest_path.open("w", encoding="utf-8", newline="\n") as handle: + json.dump(manifest, handle, indent=2, ensure_ascii=False) + handle.write("\n") + + return True + + +def main() -> int: + repo_root = _locate_repo_root(Path(__file__).parent) + cases_root = repo_root / "tests" / "reachability" / "fixtures" / "reachbench-2025-expanded" / "cases" + + updated = 0 + for case_dir in sorted([p for p in cases_root.iterdir() if p.is_dir()], key=lambda p: p.name): + images = case_dir / "images" + for variant_name in ("reachable", "unreachable"): + variant_dir = images / variant_name + if _update_manifest(variant_dir): + updated += 1 + + print(f"Updated {updated} variant manifests under {cases_root}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/tests/reachability/runners/run_all.ps1 b/tests/reachability/runners/run_all.ps1 new file mode 100644 index 000000000..f88f4bb31 --- /dev/null +++ b/tests/reachability/runners/run_all.ps1 @@ -0,0 +1,8 @@ +$ErrorActionPreference = "Stop" + +python (Join-Path $PSScriptRoot "..\\scripts\\update_corpus_manifest.py") | Out-Null +python (Join-Path $PSScriptRoot "..\\samples-public\\scripts\\update_manifest.py") | Out-Null +python (Join-Path $PSScriptRoot "..\\fixtures\\reachbench-2025-expanded\\harness\\update_variant_manifests.py") | Out-Null + +Write-Host "reachability: manifests regenerated" + diff --git a/tests/reachability/runners/run_all.sh b/tests/reachability/runners/run_all.sh new file mode 100644 index 000000000..3d0d948fb --- /dev/null +++ b/tests/reachability/runners/run_all.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +python3 "$(dirname "$0")/../scripts/update_corpus_manifest.py" >/dev/null +python3 "$(dirname "$0")/../samples-public/scripts/update_manifest.py" >/dev/null +python3 "$(dirname "$0")/../fixtures/reachbench-2025-expanded/harness/update_variant_manifests.py" >/dev/null + +echo "reachability: manifests regenerated" + diff --git a/tests/reachability/scripts/update_corpus_manifest.py b/tests/reachability/scripts/update_corpus_manifest.py index 093df423b..4efe9412e 100644 --- a/tests/reachability/scripts/update_corpus_manifest.py +++ b/tests/reachability/scripts/update_corpus_manifest.py @@ -9,7 +9,7 @@ import json from pathlib import Path ROOT = Path(__file__).resolve().parents[1] / "corpus" -FILE_LIST = ["expect.yaml", "callgraph.static.json", "vex.openvex.json"] +FILE_LIST = ["callgraph.static.json", "ground-truth.json", "vex.openvex.json"] def sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() @@ -30,7 +30,7 @@ def main() -> int: "files": files, }) manifest_path = ROOT / "manifest.json" - manifest_path.write_text(json.dumps(entries, indent=2, sort_keys=True)) + manifest_path.write_text(json.dumps(entries, indent=2, sort_keys=True) + "\n") print(f"wrote {manifest_path} ({len(entries)} entries)") return 0 diff --git a/tests/shared/OpenSslAutoInit.cs b/tests/shared/OpenSslAutoInit.cs index 5d7c3d506..7d80b38bb 100644 --- a/tests/shared/OpenSslAutoInit.cs +++ b/tests/shared/OpenSslAutoInit.cs @@ -3,7 +3,7 @@ using System.Runtime.CompilerServices; namespace StellaOps.Testing; /// -/// Automatically ensures OpenSSL 1.1 shim is visible for Mongo2Go-based tests. +/// Automatically ensures OpenSSL 1.1 shim is visible for tests that require legacy OpenSSL. /// internal static class OpenSslAutoInit { diff --git a/tests/shared/OpenSslLegacyShim.cs b/tests/shared/OpenSslLegacyShim.cs index 1e0f80789..69f5dfa20 100644 --- a/tests/shared/OpenSslLegacyShim.cs +++ b/tests/shared/OpenSslLegacyShim.cs @@ -5,7 +5,7 @@ using System.Linq; namespace StellaOps.Testing; /// -/// Ensures OpenSSL 1.1 native libraries are visible to Mongo2Go on platforms that no longer ship them. +/// Ensures OpenSSL 1.1 native libraries are visible on platforms that no longer ship them. /// public static class OpenSslLegacyShim {